diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml new file mode 100644 index 00000000000000..af172a082f39a7 --- /dev/null +++ b/.github/actions/docker-custom-build-and-push/action.yml @@ -0,0 +1,92 @@ +name: Custom Docker build and push +description: "Build and push a Docker image to Docker Hub" + +inputs: + username: + description: "Docker Hub username" + password: + description: "Docker Hub password" + publish: + description: "Set to true to actually publish the image to Docker Hub" + + context: + description: "Same as docker/build-push-action" + required: false + file: + description: "Same as docker/build-push-action" + required: false + platforms: + description: "Same as docker/build-push-action" + required: false + + images: + # e.g. linkedin/datahub-gms + description: "List of Docker images to use as base name for tags" + required: true + tags: + # e.g. latest,head,sha12345 + description: "List of tags to use for the Docker image" + required: true +outputs: + image_tag: + description: "Docker image tags" + value: ${{ steps.docker_meta.outputs.tags }} + # image_name: ${{ env.DATAHUB_GMS_IMAGE }} + +runs: + using: "composite" + + steps: + - name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + # list of Docker images to use as base name for tags + images: ${{ inputs.images }} + # add git short SHA as Docker tag + tag-custom: ${{ inputs.tags }} + tag-custom-only: true + + # Code for testing the build when not pushing to Docker Hub. + - name: Build and Load image for testing (if not publishing) + uses: docker/build-push-action@v2 + if: ${{ inputs.publish != 'true' }} + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + # TODO this only does single-platform builds in testing? + # leaving it for now since it matches the previous behavior + platforms: linux/amd64 + tags: ${{ steps.docker_meta.outputs.tags }} + load: true + push: false + - name: Upload image locally for testing (if not publishing) + uses: ishworkh/docker-image-artifact-upload@v1 + if: ${{ inputs.publish != 'true' }} + with: + image: ${{ steps.docker_meta.outputs.tags }} + + # Code for building multi-platform images and pushing to Docker Hub. + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + if: ${{ inputs.publish == 'true' }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + if: ${{ inputs.publish == 'true' }} + - name: Login to DockerHub + uses: docker/login-action@v1 + if: ${{ inputs.publish == 'true' }} + with: + username: ${{ inputs.username }} + password: ${{ inputs.password }} + - name: Build and Push Multi-Platform image + uses: docker/build-push-action@v2 + if: ${{ inputs.publish == 'true' }} + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + tags: ${{ steps.docker_meta.outputs.tags }} + push: true + + # TODO add code for vuln scanning? diff --git a/.github/actions/ensure-codegen-updated/action.yml b/.github/actions/ensure-codegen-updated/action.yml new file mode 100644 index 00000000000000..0e501bda11ede0 --- /dev/null +++ b/.github/actions/ensure-codegen-updated/action.yml @@ -0,0 +1,16 @@ +name: 'Ensure codegen is updated' +description: 'Will check the local filesystem against git, and abort if there are uncommitted changes.' + +runs: + using: "composite" + steps: + - shell: bash + run: | + if output=$(git status --porcelain) && [ ! -z "$output" ]; then + # See https://unix.stackexchange.com/a/155077/378179. + echo 'There are uncommitted changes:' + echo $output + exit 1 + else + echo 'All good!' + fi diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 48e87b4a3216ce..55ba7f82db3f3e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -10,12 +10,15 @@ on: branches: - master paths-ignore: - - "docker/**" - "docs/**" - "**.md" release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest @@ -45,15 +48,7 @@ jobs: **/build/test-results/test/** **/junit.*.xml - name: Ensure codegen is updated - run: | - if output=$(git status --porcelain) && [ ! -z "$output" ]; then - # See https://unix.stackexchange.com/a/155077/378179. - echo 'There are uncommitted changes:' - echo $output - exit 1 - else - echo 'All good!' - fi + uses: ./.github/actions/ensure-codegen-updated - name: Slack failure notification if: failure() && github.event_name == 'push' uses: kpritam/slack-job-status-action@v1 diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index f0d185a88a741b..3da2400cddc924 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -5,6 +5,7 @@ on: branches: - master paths-ignore: + - "docker/**" - "docs/**" - "**.md" pull_request: @@ -17,6 +18,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_jars: diff --git a/.github/workflows/check-quickstart.yml b/.github/workflows/check-quickstart.yml new file mode 100644 index 00000000000000..be2851bc3a1440 --- /dev/null +++ b/.github/workflows/check-quickstart.yml @@ -0,0 +1,47 @@ +name: check quickstart +on: + push: + branches: + - master + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + +jobs: + test-quickstart: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/setup-python@v2 + with: + python-version: "3.9.9" + - name: Install acryl-datahub + run: | + pip install --upgrade acryl-datahub + datahub version + python -c "import platform; print(platform.platform())" + - name: Run quickstart + run: | + datahub docker quickstart + - name: Ingest sample data + run: | + datahub docker ingest-sample-data + - name: See status + run: | + docker ps -a && datahub docker check + - name: store logs + if: failure() + run: | + docker logs datahub-gms >& quickstart-gms.log + - name: Upload logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: docker-quickstart-logs-${{ matrix.os }} + path: "*.log" \ No newline at end of file diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml new file mode 100644 index 00000000000000..efc96c25fe9661 --- /dev/null +++ b/.github/workflows/close-stale-issues.yml @@ -0,0 +1,22 @@ +name: Close inactive issues +on: + schedule: + - cron: "30 1 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: 30 + days-before-issue-close: 30 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 30 days with no activity. If you believe this is still an issue on the latest DataHub release please leave a comment with the version that you tested it with. If this is a question/discussion please head to https://slack.datahubproject.io. For feature requests please use https://feature-requests.datahubproject.io" + close-issue-message: "This issue was closed because it has been inactive for 30 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker-feast-source.yml b/.github/workflows/docker-feast-source.yml index 865d2f5cf92259..a6526c4e447887 100644 --- a/.github/workflows/docker-feast-source.yml +++ b/.github/workflows/docker-feast-source.yml @@ -3,21 +3,22 @@ on: push: branches: - master - paths-ignore: - - 'docs/**' - - '**.md' + paths: + - 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**' + - '.github/workflows/docker-feast-source.yml' pull_request: branches: - master paths: - 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**' - '.github/workflows/docker-feast-source.yml' - paths_ignore: - - '**.md' - - '**.env' release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml new file mode 100644 index 00000000000000..144cc87409eb04 --- /dev/null +++ b/.github/workflows/docker-ingestion-base.yml @@ -0,0 +1,43 @@ +name: ingestion base +on: + release: + types: [published, edited] + push: + branches: + - master + paths: + - "docker/datahub-ingestion/**" + - "gradle*" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + build-base: + name: Build and Push Docker Image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.ACRYL_DOCKER_USERNAME }} + password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} + - name: Build and Push image + uses: docker/build-push-action@v2 + with: + context: ./docker/datahub-ingestion + file: ./docker/datahub-ingestion/base.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: acryldata/datahub-ingestion-base:latest + push: true diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml new file mode 100644 index 00000000000000..a3116df793b70a --- /dev/null +++ b/.github/workflows/docker-ingestion-smoke.yml @@ -0,0 +1,42 @@ +name: ingestion smoke +on: + release: + types: [published, edited] + push: + branches: + - master + paths: + - "docker/datahub-ingestion/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + build-smoke: + name: Build and Push Docker Image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.ACRYL_DOCKER_USERNAME }} + password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} + - name: Build and Push image + uses: docker/build-push-action@v2 + with: + context: . + file: ./docker/datahub-ingestion/smoke.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: acryldata/datahub-ingestion-base:smoke + push: true diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml index d78ba2f9b11f05..a3ac1bda5d8c9a 100644 --- a/.github/workflows/docker-ingestion.yml +++ b/.github/workflows/docker-ingestion.yml @@ -10,14 +10,17 @@ on: branches: - master paths: - - "docker/**" + - "metadata-ingestion/**" + - "metadata-models/**" + - "docker/datahub-ingestion/**" - ".github/workflows/docker-ingestion.yml" - paths_ignore: - - "**.md" - - "**.env" release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest @@ -66,7 +69,7 @@ jobs: with: # list of Docker images to use as base name for tags images: | - linkedin/datahub-ingestion + heruko/datahub-ingestion # add git short SHA as Docker tag tag-custom: ${{ needs.setup.outputs.tag }} tag-custom-only: true diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml index 41bb76adb2ea37..e1c775634bcaae 100644 --- a/.github/workflows/docker-postgres-setup.yml +++ b/.github/workflows/docker-postgres-setup.yml @@ -3,9 +3,9 @@ on: push: branches: - master - paths-ignore: - - 'docs/**' - - '**.md' + paths: + - './docker/postgres-setup/**' + - '.github/workflows/docker-postgres-setup.yml' pull_request: branches: - master @@ -15,6 +15,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 708dab51044c23..34148c3a5512b9 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -15,6 +15,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: DATAHUB_GMS_IMAGE: 'heruko/datahub-gms' DATAHUB_FRONTEND_IMAGE: 'heruko/datahub-frontend-react' @@ -54,6 +58,7 @@ jobs: run: | echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}" echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}" + gms_build: name: Build and Push DataHub GMS Docker Image runs-on: ubuntu-latest @@ -66,52 +71,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_GMS_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-gms/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-gms/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true gms_scan: name: "[Monitoring] Scan GMS images for vulnerabilities" runs-on: ubuntu-latest @@ -130,6 +101,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + mae_consumer_build: name: Build and Push DataHub MAE Consumer Docker Image runs-on: ubuntu-latest @@ -142,52 +114,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-mae-consumer/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push Platform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-mae-consumer/Dockerfile platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true mae_consumer_scan: name: "[Monitoring] Scan MAE consumer images for vulnerabilities" runs-on: ubuntu-latest @@ -206,6 +144,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + mce_consumer_build: name: Build and Push DataHub MCE Consumer Docker Image runs-on: ubuntu-latest @@ -218,52 +157,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-mce-consumer/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push Platform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-mce-consumer/Dockerfile platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true mce_consumer_scan: name: "[Monitoring] Scan MCE consumer images for vulnerabilities" runs-on: ubuntu-latest @@ -282,6 +187,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + datahub_upgrade_build: name: Build and Push DataHub Upgrade Docker Image runs-on: ubuntu-latest @@ -294,52 +200,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_UPGRADE_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-upgrade/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and Push Platform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-upgrade/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + platforms: linux/amd64,linux/arm64 datahub_upgrade_scan: name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities" runs-on: ubuntu-latest @@ -358,6 +230,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + frontend_build: name: Build and Push DataHub Frontend Docker Image runs-on: ubuntu-latest @@ -370,52 +243,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_FRONTEND_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-frontend/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-frontend/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true frontend_scan: name: "[Monitoring] Scan Frontend images for vulnerabilities" runs-on: ubuntu-latest @@ -434,6 +273,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + kafka_setup_build: name: Build and Push DataHub Kafka Setup Docker Image runs-on: ubuntu-latest @@ -446,52 +286,19 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: ./docker/kafka-setup - file: ./docker/kafka-setup/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: ./docker/kafka-setup file: ./docker/kafka-setup/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + mysql_setup_build: name: Build and Push DataHub MySQL Setup Docker Image runs-on: ubuntu-latest @@ -504,52 +311,19 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/mysql-setup/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/mysql-setup/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + elasticsearch_setup_build: name: Build and Push DataHub Elasticsearch Setup Docker Image runs-on: ubuntu-latest @@ -562,52 +336,19 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/elasticsearch-setup/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/elasticsearch-setup/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + smoke_test: name: Run Smoke Tests runs-on: ubuntu-latest @@ -659,6 +400,23 @@ jobs: run: | echo "$DATAHUB_VERSION" ./smoke-test/smoke.sh + - name: store logs + if: failure() + run: | + docker ps -a + docker logs datahub-gms >& gms.log + - name: Upload logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: docker logs + path: "*.log" + - name: Upload screenshots + uses: actions/upload-artifact@v2 + if: failure() + with: + name: cypress-snapshots + path: smoke-test/tests/cypress/cypress/screenshots/ - uses: actions/upload-artifact@v2 if: always() with: diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 362e882ab97855..a8a66b628f3d70 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -10,6 +10,10 @@ on: # release: # types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: gh-pages: runs-on: ubuntu-latest diff --git a/.github/workflows/metadata-ingestion-slow.yml b/.github/workflows/metadata-ingestion-slow.yml deleted file mode 100644 index 3c020d1d277dcc..00000000000000 --- a/.github/workflows/metadata-ingestion-slow.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: metadata ingestion slow integration tests -on: - push: - branches: - - master - paths-ignore: - - "docs/**" - - "**.md" - pull_request: - branches: - - master - paths: - - "**/nifi/**" - - "**/nifi.py" - - "**/hana/**" - - "**/hana.py" - release: - types: [published, edited] - -jobs: - metadata-ingestion-slow-integration: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9"] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: ./metadata-ingestion/scripts/install_deps.sh - - name: Run metadata-ingestion slow integration tests - run: ./gradlew :metadata-ingestion:testSlowIntegration - - uses: actions/upload-artifact@v2 - if: always() - with: - name: Test Results (metadata ingestion slow integration tests) - path: | - **/build/reports/tests/test/** - **/build/test-results/test/** - **/junit.*.xml - - event-file: - runs-on: ubuntu-latest - steps: - - name: Upload - uses: actions/upload-artifact@v2 - with: - name: Event File - path: ${{ github.event_path }} diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index 2070d0586f8037..bf3b7d4695580c 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -15,12 +15,17 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: metadata-ingestion-general: runs-on: ubuntu-latest env: SPARK_VERSION: 3.0.3 + DATAHUB_TELEMETRY_ENABLED: false strategy: matrix: python-version: ["3.6", "3.9"] @@ -29,30 +34,28 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling - with: - spark-version: '3.0.3' - hadoop-version: '3.2' - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Run metadata-ingestion tests - run: ./gradlew :metadata-ingestion:build :metadata-ingestion:check + run: ./gradlew :metadata-ingestion:build :metadata-ingestion:testQuick :metadata-ingestion:check - uses: actions/upload-artifact@v2 if: always() with: - name: Test Results (metadata ingestion general) + name: Test Results (metadata ingestion ${{ matrix.python-version }} testQuick) path: | **/build/reports/tests/test/** **/build/test-results/test/** **/junit.*.xml - metadata-ingestion-by-version: + metadata-ingestion: runs-on: ubuntu-latest env: SPARK_VERSION: 3.0.3 + DATAHUB_TELEMETRY_ENABLED: false strategy: matrix: python-version: ["3.6", "3.9"] + command: ["installAirflow1", "testIntegration", "testIntegrationBatch1", "testSlowIntegration"] fail-fast: false steps: - uses: actions/checkout@v2 @@ -64,11 +67,12 @@ jobs: spark-version: '3.0.3' hadoop-version: '3.2' - name: Install dependencies - run: ./metadata-ingestion/scripts/install_deps.sh && python -m pip install --upgrade pip && pip install tox tox-gh-actions - - name: Codegen - run: ./gradlew :metadata-ingestion:codegen - - name: Run tox tests - run: cd metadata-ingestion && tox + run: ./metadata-ingestion/scripts/install_deps.sh + - name: Run metadata-ingestion tests + run: ./gradlew :metadata-ingestion:build :metadata-ingestion:${{ matrix.command }} -x:metadata-ingestion:testQuick -x:metadata-ingestion:check + - name: pip freeze show list installed + if: always() + run: source metadata-ingestion/venv/bin/activate && pip freeze - uses: actions/upload-artifact@v2 if: always() with: diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 6f936ae7cdc2d6..fd0d8dc89d8053 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -19,6 +19,10 @@ on: release: types: [published] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest @@ -47,15 +51,7 @@ jobs: **/build/test-results/test/** **/junit.*.xml - name: Ensure codegen is updated - run: | - if output=$(git status --porcelain) && [ ! -z "$output" ]; then - # See https://unix.stackexchange.com/a/155077/378179. - echo 'There are uncommitted changes:' - echo $output - exit 1 - else - echo 'All good!' - fi + uses: ./.github/actions/ensure-codegen-updated event-file: runs-on: ubuntu-latest diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml index 06f9145645477c..ad3a774462ac40 100644 --- a/.github/workflows/metadata-model.yml +++ b/.github/workflows/metadata-model.yml @@ -9,6 +9,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 0fb2f99f9dcde9..cbaf9ba3ef7a69 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -12,6 +12,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check-secret: @@ -70,4 +75,7 @@ jobs: echo signingKey=$SIGNING_KEY >> gradle.properties ./gradlew :metadata-integration:java:spark-lineage:printVersion ./gradlew :metadata-integration:java:spark-lineage:publishToMavenLocal + # Publish apache ranger plugin to maven + ./gradlew :datahub-ranger-plugin:printVersion + ./gradlew :datahub-ranger-plugin:publishMavenJavaPublicationToMavenLocal #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml index cdafb132c82459..327667f7461d93 100644 --- a/.github/workflows/spark-smoke-test.yml +++ b/.github/workflows/spark-smoke-test.yml @@ -19,6 +19,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: spark-smoke-test: runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 73c1a2d1e069ac..d22bbcfcbe66a2 100644 --- a/build.gradle +++ b/build.gradle @@ -96,6 +96,7 @@ project.ext.externalDependency = [ 'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion", // avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro` 'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1', + 'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4', 'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0', 'logbackClassic': 'ch.qos.logback:logback-classic:1.2.9', 'lombok': 'org.projectlombok:lombok:1.18.12', @@ -116,6 +117,7 @@ project.ext.externalDependency = [ 'picocli': 'info.picocli:picocli:4.5.0', 'playEhcache': 'com.typesafe.play:play-ehcache_2.12:2.7.6', 'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6', + 'playEhcache': 'com.typesafe.play:play-ehcache_2.12:2.7.6', 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.0.8', 'playDocs': 'com.typesafe.play:play-docs_2.12:2.7.6', 'playGuice': 'com.typesafe.play:play-guice_2.12:2.7.6', diff --git a/contrib/metadata-ingestion/haskell/README.md b/contrib/metadata-ingestion/haskell/README.md deleted file mode 100644 index 0342d69302fee0..00000000000000 --- a/contrib/metadata-ingestion/haskell/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# datahub Ingestion Tool - - -## Introduction - -some tool to ingestion [jdbc-database-schema] and [etl-lineage] metadata. - -i split the ingestion procedure to two part: [datahub-producer] and different [metadata-generator] - - -## Roadmap - -- [X] datahub-producer load json avro data. -- [X] add lineage-hive generator -- [X] add dataset-jdbc generator[include [mysql, mssql, postgresql, oracle] driver] -- [X] add dataset-hive generator -- [ ] *> add lineage-oracle generator -- [ ] enhance lineage-jdbc generator to lazy iterator mode. -- [ ] enchance avro parser to show error information - - - -## Quickstart -1. install nix and channel - -``` - sudo install -d -m755 -o $(id -u) -g $(id -g) /nix - curl https://nixos.org/nix/install | sh - - nix-channel --add https://nixos.org/channels/nixos-20.03 nixpkgs - nix-channel --update nixpkgs -``` - -2. [optional] you can download specified dependency in advanced, or it will automatically download at run time. - -``` - nix-shell bin/[datahub-producer].hs.nix - nix-shell bin/[datahub-producer].py.nix - ... -``` - -3. load json data to datahub - -``` - cat sample/mce.json.dat | bin/datahub-producer.hs config -``` - -4. parse hive sql to datahub -``` - ls sample/hive_*.sql | bin/lineage_hive_generator.hs | bin/datahub-producer.hs config -``` - -5. load jdbc schema(mysql, mssql, postgresql, oracle) to datahub -``` - bin/dataset-jdbc-generator.hs | bin/datahub-producer.hs config -``` - -6. load hive schema to datahub -``` - bin/dataset-hive-generator.py | bin/datahub-producer.hs config -``` - -## Reference - -- hive/presto/vertica SQL Parser - uber/queryparser [https://github.com/uber/queryparser.git] - -- oracle procedure syntax - https://docs.oracle.com/cd/E11882_01/server.112/e41085/sqlqr01001.htm#SQLQR110 - -- postgresql procedure parser - SQream/hssqlppp [https://github.com/JakeWheat/hssqlppp.git] diff --git a/contrib/metadata-ingestion/haskell/bin/datahub-producer.hs b/contrib/metadata-ingestion/haskell/bin/datahub-producer.hs deleted file mode 100755 index ae81d52902ff93..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/datahub-producer.hs +++ /dev/null @@ -1,174 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell datahub-producer.hs.nix -i runghc - -{-# LANGUAGE OverloadedStrings, FlexibleInstances, FlexibleContexts, ScopedTypeVariables #-} -{-# LANGUAGE TemplateHaskell #-} - -import System.Environment (getArgs) -import System.Directory (canonicalizePath) -import Data.Typeable (Typeable) -import Data.Functor ((<&>)) -import Control.Arrow (left, right) -import Control.Monad ((>=>), when) -import Control.Monad.IO.Class (MonadIO(..)) - -import Control.Monad.Catch (Exception, MonadThrow(..)) - -import qualified Data.ByteString.Lazy as B -import qualified Data.ByteString.Lazy.Char8 as BC -import qualified Data.Text as T -import qualified Data.Aeson as J -import Data.String.Conversions (cs) -import qualified Data.Binary as BIN - -import Data.HashMap.Strict ((!)) -import qualified Data.HashMap.Strict as MS -import qualified Data.Vector as V - - -import Control.Lens ((^?), (^..), folded, _Just) -import Data.Aeson.Lens (key, _Array, _String) - -import qualified Data.Avro.Types as A (Value(..)) -import qualified Data.Avro as A (Schema, Result(..)) -import qualified Data.Avro.Schema as AS ( - Schema(..), resultToEither, buildTypeEnvironment - , renderFullname, parseFullname, typeName, parseAvroJSON - ) --- import Data.Avro.JSON (decodeAvroJSON) -import Data.Avro.Encode (encodeAvro) -import Data.Avro.Decode (decodeAvro) --- import Data.Avro.Deriving (makeSchema) - -import Kafka.Avro ( - SchemaRegistry(..), Subject(..), SchemaId(..) - , schemaRegistry, sendSchema - , extractSchemaId, loadSchema - ) - -import Data.Conduit (ConduitT, ZipSink(..), getZipSink, runConduitRes, runConduit, bracketP, (.|), yield) -import qualified Data.Conduit.Combinators as C -import Kafka.Conduit.Sink (ProducerRecord(..), TopicName(..), ProducePartition(..), BrokerAddress(..), kafkaSink, brokersList) - -import Network.URI (parseURI) -import Network.URI.Lens (uriAuthorityLens, uriRegNameLens, uriPortLens) - -import System.Process (readProcess) - -data StringException = StringException String deriving (Typeable, Show) -instance Exception StringException - -decodeAvroJSON :: A.Schema -> J.Value -> A.Result (A.Value A.Schema) -decodeAvroJSON schema json = - AS.parseAvroJSON union env schema json - where - env = - AS.buildTypeEnvironment missing schema - missing name = - fail ("Type " <> show name <> " not in schema") - - union (AS.Union schemas) J.Null - | AS.Null `elem` schemas = - pure $ A.Union schemas AS.Null A.Null - | otherwise = - fail "Null not in union." - union (AS.Union schemas) (J.Object obj) - | null obj = - fail "Invalid encoding of union: empty object ({})." - | length obj > 1 = - fail ("Invalid encoding of union: object with too many fields:" ++ show obj) - | otherwise = - let - canonicalize name - | isBuiltIn name = name - | otherwise = AS.renderFullname $ AS.parseFullname name - branch = - head $ MS.keys obj - names = - MS.fromList [(AS.typeName t, t) | t <- V.toList schemas] - in case MS.lookup (canonicalize branch) names of - Just t -> do - nested <- AS.parseAvroJSON union env t (obj ! branch) - return (A.Union schemas t nested) - Nothing -> fail ("Type '" <> T.unpack branch <> "' not in union: " <> show schemas) - union AS.Union{} _ = - A.Error "Invalid JSON representation for union: has to be a JSON object with exactly one field." - union _ _ = - error "Impossible: function given non-union schema." - - isBuiltIn name = name `elem` [ "null", "boolean", "int", "long", "float" - , "double", "bytes", "string", "array", "map" ] - - -fromRight :: (MonadThrow m, Show a) => String -> Either a b -> m b -fromRight label = either (throwM . StringException . (label ++) . show) return - -fromJust :: (MonadThrow m, Show a) => String -> Maybe a -> m a -fromJust label = maybe (throwM . StringException $ (label ++ "value is missing") ) return - -encodeJsonWithSchema :: (MonadIO m, MonadThrow m) - => SchemaRegistry - -> Subject - -> A.Schema - -> J.Value - -> m B.ByteString -encodeJsonWithSchema sr subj schema json = do - v <- fromRight "[decodeAvroJSON]" $ AS.resultToEither $ decodeAvroJSON schema json - mbSid <- fromRight "[SchemaRegistry.sendSchema]"=<< sendSchema sr subj schema - return $ appendSchemaId v mbSid - where appendSchemaId v (SchemaId sid)= B.cons (toEnum 0) (BIN.encode sid) <> (encodeAvro v) - -decodeJsonWithSchema :: (MonadIO m, MonadThrow m) - => SchemaRegistry - -> B.ByteString - -> m J.Value -decodeJsonWithSchema sr bs = do - (sid, payload) <- maybe (throwM . StringException $ "BadPayloadNoSchemaId") return $ extractSchemaId bs - schema <- fromRight "[SchemaRegistry.loadSchema]" =<< loadSchema sr sid - J.toJSON <$> (fromRight "[Avro.decodeAvro]" $ decodeAvro schema payload) - - -parseNixJson :: FilePath -> IO J.Value -parseNixJson f = do - stdout :: String <- read <$> readProcess "nix-instantiate" ["--eval", "--expr", "builtins.toJSON (import " ++ f ++ ")"] "" - fromRight "[Aeson.eitherDecode] parse nix json" (J.eitherDecode (cs stdout)) - -main :: IO () -main = do - args <- getArgs - when (length args /= 1) $ - error " datahub-producer.hs [config-dir]" - - confDir <- canonicalizePath (head args) - putStrLn ("confDir:" <> confDir) - confJson <- parseNixJson (confDir <> "/" <> "datahub-config.nix") - -- putStrLn ("confJson: " ++ show confJson) - schema <- fromRight "[Aeson.eitherDecode] parse asvc file:" =<< - J.eitherDecode <$> B.readFile (confDir <> "/" <> "MetadataChangeEvent.avsc") - -- putStrLn ("schema: " ++ show schema) - - let - topic = "MetadataChangeEvent" - -- schema = $(makeSchema "../MetadataChangeEvent.avsc") - sandboxL = key "services".key "linkedin-datahub-pipeline".key "sandbox" - urisL = key "uris". _Array.folded._String - brokers = confJson ^.. sandboxL.key "kafka".urisL - srs = confJson ^.. sandboxL.key "schema-registry".urisL - brokers' = map (\uriText -> BrokerAddress . cs . concat $ parseURI (cs uriText) ^.. _Just.uriAuthorityLens._Just.(uriRegNameLens <> uriPortLens)) brokers - - contents <- B.getContents <&> BC.lines - sr <- schemaRegistry (cs (head srs)) - - putStrLn " ==> beginning to send data..." - runConduitRes $ C.yieldMany contents - .| C.mapM (fromRight "[JSON.eitherDecode] read json record:". J.eitherDecode) - -- .| C.iterM (liftIO . putStrLn. cs . J.encode) - .| C.mapM (encodeJsonWithSchema sr (Subject (topic <> "-value")) schema) - -- .| C.iterM (decodeJsonWithSchema sr >=> liftIO . print . J.encode) - .| C.map (mkRecord (TopicName topic)) - .| getZipSink ( ZipSink (kafkaSink (brokersList brokers')) *> - ZipSink ((C.length >>= yield) .| C.iterM (\n -> liftIO $ putStrLn ("total table num:" <> show n)) .| C.sinkNull)) - return () - where - mkRecord :: TopicName -> B.ByteString -> ProducerRecord - mkRecord topic bs = ProducerRecord topic UnassignedPartition Nothing (Just (cs bs)) diff --git a/contrib/metadata-ingestion/haskell/bin/datahub-producer.hs.nix b/contrib/metadata-ingestion/haskell/bin/datahub-producer.hs.nix deleted file mode 100644 index 57e131e055454f..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/datahub-producer.hs.nix +++ /dev/null @@ -1,16 +0,0 @@ -with import {} ; -let -in -mkShell { - buildInputs = [ - (haskellPackages.ghcWithPackages ( p: - [ p.bytestring p.string-conversions - p.exceptions - p.network-uri p.directory - p.lens p.aeson p.lens-aeson p.avro p.hw-kafka-avro - p.hw-kafka-client - p.conduit p.hw-kafka-conduit - ] - )) - ]; -} diff --git a/contrib/metadata-ingestion/haskell/bin/dataset-hive-generator.py b/contrib/metadata-ingestion/haskell/bin/dataset-hive-generator.py deleted file mode 100755 index 9da5ae198017a4..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/dataset-hive-generator.py +++ /dev/null @@ -1,66 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell dataset-hive-generator.py.nix -i python - -import sys -import time -from pyhive import hive -from TCLIService.ttypes import TOperationState - -import simplejson as json - -HIVESTORE='localhost' - -AVROLOADPATH = '../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc' -KAFKATOPIC = 'MetadataChangeEvent_v4' -BOOTSTRAP = 'localhost:9092' -SCHEMAREGISTRY = 'http://localhost:8081' - -def hive_query(query): - """ - Execute the query to the HiveStore. - """ - cursor = hive.connect(HIVESTORE).cursor() - cursor.execute(query, async_=True) - status = cursor.poll().operationState - while status in (TOperationState.INITIALIZED_STATE, TOperationState.RUNNING_STATE): - logs = cursor.fetch_logs() - for message in logs: - sys.stdout.write(message) - status = cursor.poll().operationState - results = cursor.fetchall() - return results - -def build_hive_dataset_mce(dataset_name, schema, metadata): - """ - Create the MetadataChangeEvent via dataset_name and schema. - """ - actor, type, created_time, upstreams_dataset, sys_time = "urn:li:corpuser:" + metadata[2][7:], str(metadata[-1][11:-1]), int(metadata[3][12:]), metadata[-28][10:], int(time.time()) - owners = {"owners":[{"owner":actor,"type":"DATAOWNER"}],"lastModified":{"time":sys_time,"actor":actor}} - upstreams = {"upstreams":[{"auditStamp":{"time":sys_time,"actor":actor},"dataset":"urn:li:dataset:(urn:li:dataPlatform:hive," + upstreams_dataset + ",PROD)","type":"TRANSFORMED"}]} - elements = {"elements":[{"url":HIVESTORE,"description":"sample doc to describe upstreams","createStamp":{"time":sys_time,"actor":actor}}]} - schema_name = {"schemaName":dataset_name,"platform":"urn:li:dataPlatform:hive","version":0,"created":{"time":created_time,"actor":actor}, - "lastModified":{"time":sys_time,"actor":actor},"hash":"","platformSchema":{"com.linkedin.pegasus2avro.schema.OtherSchema": {"rawSchema": schema}}, - "fields":[{"fieldPath":"","description":{"string":""},"nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}}]} - - mce = {"auditHeader": None, - "proposedSnapshot":{"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": - {"urn": "urn:li:dataset:(urn:li:dataPlatform:hive,"+ dataset_name +",PROD)" - ,"aspects": [ - {"com.linkedin.pegasus2avro.common.Ownership": owners} - , {"com.linkedin.pegasus2avro.dataset.UpstreamLineage": upstreams} - , {"com.linkedin.pegasus2avro.common.InstitutionalMemory": elements} - , {"com.linkedin.pegasus2avro.schema.SchemaMetadata": schema_name} - ]}}, - "proposedDelta": None} - - print(json.dumps(mce)) - -databases = hive_query('show databases') -for database in databases: - tables = hive_query('show tables in ' + database[0]) - for table in tables: - dataset_name = database[0] + '.' + table[0] - description = hive_query('describe extended ' + dataset_name) - build_hive_dataset_mce(dataset_name, str(description[:-1][:-1]), description[-1][1].split(',')) - -sys.exit(0) diff --git a/contrib/metadata-ingestion/haskell/bin/dataset-hive-generator.py.nix b/contrib/metadata-ingestion/haskell/bin/dataset-hive-generator.py.nix deleted file mode 100644 index 9bfce06fdd775d..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/dataset-hive-generator.py.nix +++ /dev/null @@ -1,61 +0,0 @@ -with import {} ; -let - avro-python3-1_8 = python3Packages.buildPythonPackage rec { - pname = "avro-python3" ; - version = "1.8.2" ; - - src = python3Packages.fetchPypi { - inherit pname version ; - sha256 = "f82cf0d66189600b1e6b442f650ad5aca6c189576723dcbf6f9ce096eab81bd6" ; - } ; - doCheck = false; - } ; - - sasl = python3Packages.buildPythonPackage rec { - pname = "sasl" ; - version = "0.2.1" ; - - src = python3Packages.fetchPypi { - inherit pname version ; - sha256 = "04f22e17bbebe0cd42471757a48c2c07126773c38741b1dad8d9fe724c16289d" ; - } ; - doCheck = false; - propagatedBuildInputs = [ cyrus_sasl ] ++ (with python3Packages ; [six]) ; - } ; - - thrift_sasl = python3Packages.buildPythonPackage rec { - pname = "thrift_sasl" ; - version = "0.4.2" ; - - src = python3Packages.fetchPypi { - inherit pname version ; - sha256 = "6a1c54731cb3ce61bdc041d9dc36e21f0f56db0163bb7b57be84de3fda70922f" ; - } ; - doCheck = false; - propagatedBuildInputs = with python3Packages; [ thrift sasl ] ; - } ; - - PyHive = python3Packages.buildPythonPackage rec { - pname = "PyHive" ; - version = "0.6.1" ; - - src = python3Packages.fetchPypi { - inherit pname version ; - sha256 = "a5f2b2f8bcd85a8cd80ab64ff8fbfe1c09515d266650a56f789a8d89ad66d7f4" ; - } ; - doCheck = false; - propagatedBuildInputs = with python3Packages ; [ dateutil future thrift sasl thrift_sasl ]; - } ; - -in -mkShell { - buildInputs = (with python3Packages ;[ - python - requests - PyHive - - simplejson - # avro-python3-1_8 - # confluent-kafka - ]) ; -} diff --git a/contrib/metadata-ingestion/haskell/bin/dataset-jdbc-generator.hs b/contrib/metadata-ingestion/haskell/bin/dataset-jdbc-generator.hs deleted file mode 100755 index 68748f38ddbbb3..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/dataset-jdbc-generator.hs +++ /dev/null @@ -1,213 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell dataset-jdbc-generator.hs.nix -i "runghc --ghc-arg=-fobject-code" - -{-# LANGUAGE OverloadedStrings, FlexibleInstances, FlexibleContexts, ScopedTypeVariables #-} -{-# LANGUAGE DataKinds #-} -{-# LANGUAGE TemplateHaskell, QuasiQuotes #-} -{-# LANGUAGE TypeOperators #-} - - -{-# OPTIONS_GHC -fplugin=Language.Java.Inline.Plugin #-} - - -import System.Environment (lookupEnv) -import System.IO (hPrint, stderr, hSetEncoding, stdout, utf8) - -import qualified Language.Haskell.TH.Syntax as TH - -import Control.Concurrent (runInBoundThread) -import Language.Java (J, withJVM, reify, reflect, JType(..)) -import Language.Java.Inline (imports, java) - -import qualified Data.Text as T -import qualified Data.Text.IO as T -import Data.String.Conversions (cs) -import Text.InterpolatedString.Perl6 (q) - -import Prelude hiding ((>>=), (>>)) - -import Data.Conduit (ConduitT, ZipSink(..), getZipSink, runConduitRes, runConduit, bracketP, (.|), yield) -import qualified Data.Conduit.Combinators as C -import qualified Data.Conduit.List as C (groupBy) - -import qualified Data.Aeson as J -import Control.Arrow ((>>>)) -import Data.Aeson.QQ (aesonQQ) - -import Text.Printf (printf) - - -imports "java.util.*" -imports "java.sql.*" - - -datasetOracleSql :: T.Text -datasetOracleSql = [q| - select - c.OWNER || '.' || c.TABLE_NAME as schema_name - , t.COMMENTS as schema_description - , c.COLUMN_NAME as field_path - , c.DATA_TYPE as native_data_type - , m.COMMENTS as description - from ALL_TAB_COLUMNS c - left join ALL_TAB_COMMENTS t - on c.OWNER = t.OWNER - and c.TABLE_NAME = t.TABLE_NAME - left join ALL_COL_COMMENTS m - on c.OWNER = m.OWNER - and c.TABLE_NAME = m.TABLE_NAME - and c.COLUMN_NAME = m.COLUMN_NAME - where NOT REGEXP_LIKE(c.OWNER, 'ANONYMOUS|PUBLIC|SYS|SYSTEM|DBSNMP|MDSYS|CTXSYS|XDB|TSMSYS|ORACLE.*|APEX.*|TEST?*|GG_.*|\\$') - order by schema_name, c.COLUMN_ID -|] - -datasetMysqlSql :: T.Text -datasetMysqlSql = [q| - select - concat(c.TABLE_SCHEMA, '.', c.TABLE_NAME) as schema_name - , NULL as schema_description - , c.COLUMN_NAME as field_path - , c.DATA_TYPE as native_data_type - , c.COLUMN_COMMENT as description - from information_schema.columns c - where table_schema not in ('information_schema') - order by schema_name, c.ORDINAL_POSITION -|] - -datasetPostgresqlSql :: T.Text -datasetPostgresqlSql = [q| - SELECT - c.table_schema || '.' || c.table_name as schema_name - , pgtd.description as schema_description - , c.column_name as field_path - , c.data_type as native_data_type - , pgcd.description as description - FROM INFORMATION_SCHEMA.COLUMNS c - INNER JOIN - pg_catalog.pg_statio_all_tables as st on c.table_schema=st.schemaname and c.table_name=st.relname - LEFT JOIN - pg_catalog.pg_description pgcd on pgcd.objoid=st.relid and pgcd.objsubid=c.ordinal_position - LEFT JOIN - pg_catalog.pg_description pgtd on pgtd.objoid=st.relid and pgtd.objsubid=0 - WHERE c.table_schema NOT IN ('information_schema', 'pg_catalog') - ORDER by schema_name, ordinal_position ; -|] - - -mkMCE :: Int -> T.Text -> [[T.Text]] -> J.Value -mkMCE ts platform fields@((schemaName:schemaDescription:_):_) = [aesonQQ| - { "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": #{urn} - , "aspects": [ - { "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{"owner": "urn:li:corpuser:datahub", "type":"DATAOWNER"}] - , "lastModified": {"time": #{ts}, "actor": "urn:li:corpuser:datahub"} - } - } - , { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": {"string": #{schemaDescription}} - } - } - , { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": #{schemaName} - , "platform": "urn:li:dataPlatform:" - , "version": 0 - , "created": {"time": #{ts}, "actor": "urn:li:corpuser:datahub"} - , "lastModified": {"time": #{ts}, "actor": "urn:li:corpuser:datahub"} - , "hash": "" - , "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "documentSchema": "{}" - , "tableSchema": "{}" - } - } - , "fields": #{mkFields fields} - } - } - ] - } - } - } - |] - where - urn :: String = printf "urn:li:dataset:(urn:li:dataPlatform:%s,%s,%s)" - platform schemaName ("PROD"::String) - mkField (_:_:fieldPath:nativeDataType:description:[]) = [aesonQQ| - { - "fieldPath": #{fieldPath} - , "description": {"string": #{description}} - , "type": {"type": {"com.linkedin.pegasus2avro.schema.StringType": {}}} - , "nativeDataType": #{nativeDataType} - } - |] - mkFields = map mkField - -main :: IO () -main = do - hSetEncoding stdout utf8 - let - jvmArgs = case $(TH.lift =<< TH.runIO (lookupEnv "CLASSPATH")) of - Nothing -> [] - Just cp -> [ cs ("-Djava.class.path=" ++ cp) ] - platform :: T.Text = "localhost_postgresql" - -- dbUrl :: T.Text = "jdbc:mysql://localhost:3306/datahub?useSSL=false" - -- dbUrl :: T.Text = "jdbc:oracle:thin@localhost:1521:EDWDB" - dbUrl :: T.Text = "jdbc:postgresql://localhost:5432/datahub" - - dbUser :: T.Text = "datahub" - dbPassword :: T.Text = "datahub" - - -- dbDriver:: T.Text = "oracle.jdbc.OracleDriver" ; - -- dbDriver:: T.Text = "com.mysql.jdbc.Driver" ; - dbDriver:: T.Text = "org.postgresql.Driver" ; - -- dbDriver:: T.Text = "com.microsoft.sqlserver.jdbc.SQLServerDriver" ; - - -- dbSQL :: T.Text = datasetMysqlSql - -- dbSQL :: T.Text = datasetOracleSql - dbSQL :: T.Text = datasetPostgresqlSql - runInBoundThread $ withJVM jvmArgs $ do - [jDbUrl, jDbUser, jDbPassword, jDbDriver, jDbSQL ] <- - mapM reflect [dbUrl, dbUser, dbPassword, dbDriver, dbSQL] - - result <- [java| { - try { - Class.forName($jDbDriver) ; - } catch (ClassNotFoundException e) { - e.printStackTrace() ; - System.exit(1) ; - } - - List result = new ArrayList() ; - try (Connection con = DriverManager.getConnection($jDbUrl, $jDbUser, $jDbPassword) ; - Statement st = con.createStatement(); ) { - try (ResultSet rs = st.executeQuery($jDbSQL)) { - while(rs.next()) { - String[] row = new String[] { - Optional.ofNullable(rs.getString("schema_name")).orElse("") - , Optional.ofNullable(rs.getString("schema_description")).orElse("") - , Optional.ofNullable(rs.getString("field_path")).orElse("") - , Optional.ofNullable(rs.getString("native_data_type")).orElse("") - , Optional.ofNullable(rs.getString("description")).orElse("") - } ; - result.add(row) ; - } - } - return result.toArray(new String[0][0]) ; - } catch (SQLException e) { - e.printStackTrace() ; - return null ; - } - } |] - - rows :: [[T.Text]] <- reify result - runConduit $ C.yieldMany rows - -- .| C.iterM (hPrint stderr) - .| C.groupBy sameSchemaName - -- .| C.iterM (hPrint stderr) - .| C.map (mkMCE 0 platform) - .| C.mapM_ (J.encode >>> cs >>> putStrLn) - .| C.sinkNull - return () - where - sameSchemaName (schemaNameL:_) (schemaNameR:_) = schemaNameL == schemaNameR diff --git a/contrib/metadata-ingestion/haskell/bin/dataset-jdbc-generator.hs.nix b/contrib/metadata-ingestion/haskell/bin/dataset-jdbc-generator.hs.nix deleted file mode 100644 index c7f48e0163be14..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/dataset-jdbc-generator.hs.nix +++ /dev/null @@ -1,54 +0,0 @@ -with import {} ; -let - inline_java_git = fetchFromGitHub { - owner = "tweag" ; - repo = "inline-java" ; - rev = "a897d32df99e4ed19314d2a7e245785152e9099d" ; - sha256 = "00pk19j9g0mm9sknj3aklz01zv1dy234s3vnzg6daq1dmwd4hb68" ; - } ; - haskellPackages = pkgs.haskellPackages.override { - overrides = self: super: with pkgs.haskell.lib; { - jni = overrideCabal (self.callCabal2nix "jni" (inline_java_git + /jni) {}) (drv: { - preConfigure = '' - local libdir=( "${pkgs.jdk}/lib/openjdk/jre/lib/"*"/server" ) - configureFlags+=" --extra-lib-dir=''${libdir[0]}" - '' ; - }) ; - - jvm = overrideCabal (self.callCabal2nix "jvm" (inline_java_git + /jvm) {}) (drv: { - doCheck = false ; - }) ; - inline-java = overrideCabal (self.callCabal2nix "inline-java" inline_java_git {}) (drv: { - doCheck = false ; - }) ; - jvm-batching = overrideCabal (self.callCabal2nix "jvm-batching" (inline_java_git + /jvm-batching) {}) (drv: { - doCheck = false ; - }) ; - jvm-streaming = overrideCabal (self.callCabal2nix "jvm-streaming" (inline_java_git + /jvm-streaming) {}) (drv: { - doCheck = false ; - }) ; - - } ; - }; - -in -mkShell { - buildInputs = [ - pkgs.jdk - pkgs.postgresql_jdbc - pkgs.mysql_jdbc - pkgs.mssql_jdbc - pkgs.oracle-instantclient - - (haskellPackages.ghcWithPackages ( p: - [ p.bytestring p.string-conversions - p.interpolatedstring-perl6 - p.aeson p.aeson-qq - p.exceptions - p.inline-java - - p.conduit - ] - )) - ]; -} diff --git a/contrib/metadata-ingestion/haskell/bin/lineage_hive_generator.hs b/contrib/metadata-ingestion/haskell/bin/lineage_hive_generator.hs deleted file mode 100755 index a8549f39da0a98..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/lineage_hive_generator.hs +++ /dev/null @@ -1,113 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell ./lineage_hive_generator.hs.nix -i runghc - -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE TemplateHaskell #-} -{-# LANGUAGE QuasiQuotes #-} - -import Data.Functor ((<&>)) -import Control.Monad (when) -import Control.Arrow ((>>>)) -import Data.Proxy (Proxy(..)) -import Data.Either (isLeft, fromLeft, fromRight) - -import Text.Printf (formatString) - -import System.IO (hPrint, stderr) - -import Data.String.Conversions (cs) -import qualified Data.Text.Lazy as T -import qualified Data.Text.Lazy.IO as T - -import qualified Data.Map as M -import qualified Data.Set as S -import qualified Data.HashMap.Strict as HM -import qualified Data.Aeson as J - -import Data.Conduit (ConduitT, runConduitRes, runConduit, bracketP, (.|)) -import qualified Data.Conduit.Combinators as C - -import qualified Database.Sql.Hive.Parser as HIVE -import qualified Database.Sql.Hive.Type as HIVE - -import Database.Sql.Type ( - Catalog(..), DatabaseName(..), FullyQualifiedTableName(..), FQTN(..) - , makeDefaultingCatalog, mkNormalSchema - ) - -import Database.Sql.Util.Scope (runResolverWarn) -import Database.Sql.Util.Lineage.Table (getTableLineage) - -import Data.Aeson.QQ (aesonQQ) -import Data.Time.Clock.POSIX (getPOSIXTime) - - -instance J.ToJSON FullyQualifiedTableName -instance J.ToJSONKey FullyQualifiedTableName - -nowts :: IO Int -nowts = floor . (* 1000) <$> getPOSIXTime - -catalog :: Catalog -catalog = makeDefaultingCatalog HM.empty - [mkNormalSchema "public" ()] - (DatabaseName () "defaultDatabase") - -tableName :: FullyQualifiedTableName -> T.Text -tableName (FullyQualifiedTableName database schema name) = T.intercalate "." [database, schema, name] - -mkMCE :: Int -> (FQTN, S.Set FQTN) -> J.Value -mkMCE ts (output, inputs) = [aesonQQ| - { "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": #{uriName output} - , "aspects": [ - { "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": #{upstreams ts inputs} - } - } - ] - } - } - } - |] - where - upstream :: Int -> T.Text -> J.Value - upstream ts dataset = [aesonQQ| - { "auditStamp": {"time":#{ts}, "actor":"urn:li:corpuser:jdoe"} - , "dataset": #{dataset} - , "type":"TRANSFORMED" - } - |] - upstreams ts = map (upstream ts . uriName) . S.toList - uriName :: FQTN -> T.Text - uriName fqtn = "urn:li:dataset:(urn:li:dataPlatform:hive," - <> tableName fqtn - <> ",PROD)" - - -main = do - contents <- T.getContents <&> T.lines - ts <- nowts - - runConduit $ C.yieldMany contents - .| C.iterM (hPrint stderr) - .| C.mapM (cs >>> T.readFile) - .| C.concatMap parseSQL - .| C.mapM resolveStatement - .| C.concatMap (getTableLineage >>> M.toList) - .| C.map (mkMCE ts) - .| C.mapM_ (J.encode >>> cs >>> putStrLn) - where - parseSQL sql = do - let stOrErr = HIVE.parseManyAll sql - when (isLeft stOrErr) $ - error $ show (fromLeft undefined stOrErr) - fromRight undefined stOrErr - resolveStatement st = do - let resolvedStOrErr = runResolverWarn (HIVE.resolveHiveStatement st) HIVE.dialectProxy catalog - when (isLeft . fst $ resolvedStOrErr) $ - error $ show (fromLeft undefined (fst resolvedStOrErr)) - let (Right queryResolved, resolutions) = resolvedStOrErr - return queryResolved - diff --git a/contrib/metadata-ingestion/haskell/bin/lineage_hive_generator.hs.nix b/contrib/metadata-ingestion/haskell/bin/lineage_hive_generator.hs.nix deleted file mode 100644 index 17e5883a46185f..00000000000000 --- a/contrib/metadata-ingestion/haskell/bin/lineage_hive_generator.hs.nix +++ /dev/null @@ -1,31 +0,0 @@ -with import {} ; -let - queryparser_git = fetchFromGitHub { - owner = "uber" ; - repo = "queryparser" ; - rev = "6015e8f273f4498326fec0315ac5580d7036f8a4" ; - sha256 = "05pnifm5awyqxi6330v791b1cvw26xbcn2r20pqakvl8d3xyaxa4" ; - } ; - haskellPackages = pkgs.haskellPackages.override { - overrides = self: super: with pkgs.haskell.lib; { - queryparser = appendConfigureFlag - (dontHaddock (doJailbreak (self.callCabal2nix "queryparser" queryparser_git {}))) - "--ghc-options=-XNoMonadFailDesugaring" ; - queryparser-hive = dontHaddock (doJailbreak (self.callCabal2nix "queryparser-hive" (queryparser_git + /dialects/hive) {})) ; - } ; - }; - -in -mkShell { - buildInputs = [ - (haskellPackages.ghcWithPackages ( p: - [ p.bytestring p.text p.string-conversions - p.exceptions p.time - p.aeson p.aeson-qq - p.conduit - p.queryparser p.queryparser-hive - ] - )) - ]; -} - diff --git a/contrib/metadata-ingestion/haskell/config/MetadataChangeEvent.avsc b/contrib/metadata-ingestion/haskell/config/MetadataChangeEvent.avsc deleted file mode 120000 index d5576bd9d8c723..00000000000000 --- a/contrib/metadata-ingestion/haskell/config/MetadataChangeEvent.avsc +++ /dev/null @@ -1 +0,0 @@ -../../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc \ No newline at end of file diff --git a/contrib/metadata-ingestion/haskell/config/datahub-config.nix b/contrib/metadata-ingestion/haskell/config/datahub-config.nix deleted file mode 120000 index 80fbf896760fc8..00000000000000 --- a/contrib/metadata-ingestion/haskell/config/datahub-config.nix +++ /dev/null @@ -1 +0,0 @@ -../../nix/datahub-config.nix \ No newline at end of file diff --git a/contrib/metadata-ingestion/haskell/sample/hive_1.sql b/contrib/metadata-ingestion/haskell/sample/hive_1.sql deleted file mode 100644 index 8d9eb94bcb8347..00000000000000 --- a/contrib/metadata-ingestion/haskell/sample/hive_1.sql +++ /dev/null @@ -1,144 +0,0 @@ -WITH TL AS -( - SELECT B1.CPC, - B1.CSC, - B1.CSN, - B1.CSC_P, - B1.CCC, - B1.NSD, - CASE - WHEN B1.CSL = 4 THEN - B5.CSN - ELSE - B1.CSN - END AS GL_ATTR, - B1.CSL, - CASE WHEN B1.CSL = 4 THEN B4.CSN - WHEN B1.CSL = 3 THEN B3.CSN - WHEN B1.CSL = 2 THEN B2.CSN - WHEN B1.CSL = 1 THEN B1.CSN - END AS FACCTNAME_LV1, - CASE WHEN B1.CSL = 4 THEN B3.CSN - WHEN B1.CSL = 3 THEN B2.CSN - WHEN B1.CSL = 2 THEN B1.CSN - END AS FACCTNAME_LV2, - CASE WHEN B1.CSL = 4 THEN B2.CSN - WHEN B1.CSL = 3 THEN B1.CSN - END AS FACCTNAME_LV3, - CASE WHEN B1.CSL = 4 THEN B1.CSN - END AS FACCTNAME_LV4 - FROM (SELECT CPC, CSC, CSN, CSC_P, CCC - , NSD, CSL - FROM ODS.FVS - WHERE HDATASRC = 'A' - ) B1 - LEFT JOIN - (SELECT CPC, CSC, CSN, CSC_P, CCC - , NSD, CSL - FROM ODS.FVS - WHERE HDATASRC = 'A' - ) B2 - ON B1.CPC = B2.CPC - AND B1.CSC_P = B2.CSC - LEFT JOIN - (SELECT CPC, CSC, CSN, CSC_P, CCC - , NSD, CSL - FROM ODS.FVS - WHERE HDATASRC = 'A' - ) B3 - ON B2.CPC = B3.CPC - AND B2.CSC_P = B3.CSC - LEFT JOIN - (SELECT CPC, CSC, CSN, CSC_P, CCC - , NSD, CSL - FROM ODS.FVS - WHERE HDATASRC = 'A' - ) B4 - ON B3.CPC = B4.CPC - AND B3.CSC_P = B4.CSC - LEFT JOIN - (SELECT CPC, CSC, CSN, CSC_P, CCC - , NSD, CSL - FROM ODS.FVS - WHERE HDATASRC = 'A' - ) B5 - ON B1.CPC = B5.CPC - AND B1.CSC_P = B5.CSC -) -INSERT OVERWRITE TABLE TMP.TFVDM1 PARTITION (HDATASRC = 'A') -SELECT qt_sequence("UUID", A.CAC) AS UUID, - C.PH AS PH, - A.CAC AS PC, - A.CPC AS ASS, - A.D_BIZ AS BD, - E.CH AS CH, - F.EH AS EH, - A.CSC AS GL_CODE, - CASE - WHEN A.CSN = ' ' THEN - A.C_KEY_NAME - ELSE - NVL(A.CSN,A.C_KEY_NAME) - END AS GL_NAME, - A.N_VALPRICE AS ATPRICE, - A.N_HLDAMT AS ATQTY, - A.N_HLDCST_LOCL AS ATCOST, - A.N_HLDCST AS ATCOST_ORICUR, - A.N_HLDMKV_LOCL AS ATMKTVAL, - A.N_HLDMKV AS ATMKTVAL_ORICUR, - A.N_HLDVVA_L AS ATVAL_ADDED, - A.N_HLDVVA AS ATVAL_ADDED_ORICUR, - A.N_VALRATE AS ATEXRATE, - NULL AS COST_TO_AT_RIO, - NULL AS MKTVAL_TO_AT_RIO, - B.NSD AS IS_DETAIL_GL, - A.C_PA_CODE AS ATITEM, - A.C_IVT_CLSS AS INVEST_CLASS, - A.C_ML_ATTR AS ISSUE_MODE, - A.C_FEE_CODE AS FEE_CODE, - A.C_SEC_VAR_MX AS SEC_KIND, - A.C_TD_ATTR AS TRADE_ATTR, - H.C_CA_ATTR AS CASH_ACCOUNT, - A.GL_LV1 AS GL_LV1, - B.FACCTNAME_LV1 AS GL_NAME_LV1, - B.FACCTNAME_LV2 AS GL_NAME_LV2, - B.FACCTNAME_LV3 AS GL_NAME_LV3, - B.FACCTNAME_LV4 AS GL_NAME_LV4, - NULL AS GL_NAME_LV5, - NULL AS GL_NAME_LV6, - A.CSC_T AS GL_ATTR_CODE, - CASE WHEN B.GL_ATTR = '' THEN A.CSN - ELSE B.GL_ATTR END AS GL_ATTR, - NVL(B.CSN, A.C_KEY_NAME) AS GL_FNAME, - A.C_SEC_CODE AS SEC_CODE_FA, - NULL AS SYMBOL_ORI, - NULL AS SYMBOL, - NULL AS SEC_TYPE , - FROM_UNIXTIME(UNIX_TIMESTAMP(CURRENT_TIMESTAMP()),'yyyy-MM-dd HH:mm:ss') AS HLOADTIME, - '20190101' AS HBATCHDATE - FROM (SELECT SUBSTR(T.CSC, 1, 4) AS GL_LV1, - T.* - FROM ODS.FVRV T - WHERE T.D_BIZ IN (SELECT BD - FROM CTL.CFD - WHERE HDATASRC = 'A') - AND T.HDATASRC = 'A' - ) A - LEFT JOIN TL B - ON NVL(A.CSC_T, A.CSC) = B.CSC - AND A.CPC = B.CPC - LEFT JOIN DW.PPCM C - ON A.CPC = C.ORI_SYS_PC - AND C.ORI_SYS_HCODE = 'A' - AND A.D_BIZ BETWEEN C.STDATE AND C.ENDDATE - LEFT JOIN DW.RCM E - ON A.CCC = E.ORI_SYS_CR_CODE - AND E.ORI_SYS_HCODE = 'A' - LEFT JOIN DW.REM F - ON A.C_MKT_CODE = F.ORI_SYS_EXCH_CODE - AND F.ORI_SYS_HCODE = 'A' - LEFT JOIN (SELECT C_CA_CODE, MAX(C_CA_ATTR) AS C_CA_ATTR - FROM ODS.FVC - GROUP BY C_CA_CODE) H - ON A.C_CA_CODE = H.C_CA_CODE - diff --git a/contrib/metadata-ingestion/haskell/sample/mce.json.dat b/contrib/metadata-ingestion/haskell/sample/mce.json.dat deleted file mode 100644 index da70cd52418d56..00000000000000 --- a/contrib/metadata-ingestion/haskell/sample/mce.json.dat +++ /dev/null @@ -1,5 +0,0 @@ -{"proposedSnapshot": {"com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": {"urn": "urn:li:corpuser:datahub", "aspects": [{"com.linkedin.pegasus2avro.identity.CorpUserInfo":{"active": true, "displayName": {"string": "Data Hub"}, "email": "datahub@linkedin.com", "title": {"string": "CEO"}, "fullName": {"string": "Data Hub"}}}]}}} -{"proposedSnapshot": {"com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": {"urn": "urn:li:corpuser:jdoe", "aspects": [{"com.linkedin.pegasus2avro.identity.CorpUserInfo":{"active": true, "displayName": {"string": "John Doe"}, "email": "jdoe@linkedin.com", "title": {"string": "Software Engineer"}, "fullName": {"string": "John Doe"}}}]}}} -{"proposedSnapshot": {"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {"urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", "aspects": [{"com.linkedin.pegasus2avro.common.Ownership": {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}},{"com.linkedin.pegasus2avro.dataset.UpstreamLineage":{"upstreams":[]}}, {"com.linkedin.pegasus2avro.common.InstitutionalMemory":{"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}},{"com.linkedin.pegasus2avro.schema.SchemaMetadata":{"schemaName":"SampleKafkaSchema","platform":"urn:li:dataPlatform:kafka","version":0, "created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}, "lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}, "hash":"","platformSchema":{"com.linkedin.pegasus2avro.schema.KafkaSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset.\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"}}, "fields":[{"fieldPath":"field_foo", "description":{"string": "Foo field description"},"type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}},"nativeDataType":"string"}, {"fieldPath":"field_bar", "description":{"string": "Bar field description"},"type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}},"nativeDataType":"boolean"}]}}] } } } -{"proposedSnapshot": {"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {"urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", "aspects": [{"com.linkedin.pegasus2avro.common.Ownership": {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}},{"com.linkedin.pegasus2avro.dataset.UpstreamLineage":{"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)","type":"TRANSFORMED"}]}}, {"com.linkedin.pegasus2avro.common.InstitutionalMemory":{"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}},{"com.linkedin.pegasus2avro.schema.SchemaMetadata":{"schemaName":"SampleHdfsSchema","platform":"urn:li:dataPlatform:hdfs","version":0, "created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}, "lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}, "hash":"","platformSchema":{"com.linkedin.pegasus2avro.schema.KafkaSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset.\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"}}, "fields":[{"fieldPath":"field_foo", "description":{"string": "Foo field description"},"type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}},"nativeDataType":"string"}, {"fieldPath":"field_bar", "description":{"string": "Bar field description"},"type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}},"nativeDataType":"boolean"}]}}] } } } -{"proposedSnapshot": {"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {"urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspects": [{"com.linkedin.pegasus2avro.common.Ownership": {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}},{"com.linkedin.pegasus2avro.dataset.UpstreamLineage":{"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)","type":"TRANSFORMED"}]}}, {"com.linkedin.pegasus2avro.common.InstitutionalMemory":{"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}},{"com.linkedin.pegasus2avro.schema.SchemaMetadata":{"schemaName":"SampleHiveSchema","platform":"urn:li:dataPlatform:hive","version":0, "created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}, "lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}, "hash":"","platformSchema":{"com.linkedin.pegasus2avro.schema.KafkaSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset.\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"}}, "fields":[{"fieldPath":"field_foo", "description":{"string": "Foo field description"},"type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}},"nativeDataType":"string"}, {"fieldPath":"field_bar", "description":{"string": "Bar field description"},"type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}},"nativeDataType":"boolean"}]}}] } } } \ No newline at end of file diff --git a/contrib/metadata-ingestion/python/looker/dashboard_ingestion/README.md b/contrib/metadata-ingestion/python/looker/dashboard_ingestion/README.md deleted file mode 100644 index dd70429597e6b7..00000000000000 --- a/contrib/metadata-ingestion/python/looker/dashboard_ingestion/README.md +++ /dev/null @@ -1,15 +0,0 @@ -## looker_dashboard_ingestion.py -This tool helps ingest Looker dashboard and chart metadata into datahub. -Currently it creates a separate platform named "looker" and loads all dashboard and chart information into that platform as virtual datasets. This was to workaround datahub's lack of support for dashboard entities, however datahub recently started supporting proper dashboard entities. - -The script assumes you already have run lookml_ingestion.py to scrape view definitions into datahub, this is important because we assign lineage between looker views and looker dashboards/charts where possible. - - -## Steps: -- Use a version of python >= 3.7 -- Make a virtual environment -- pip install -r requirements.txt -- Set env vars: LOOKERSDK_CLIENT_ID, LOOKERSDK_CLIENT_SECRET, LOOKERSDK_BASE_URL -- Configure extra kafka conf in looker_dashboard_ingestion.py - -python looker_dashboard_ingestion.py \ No newline at end of file diff --git a/contrib/metadata-ingestion/python/looker/dashboard_ingestion/looker_dashboard_ingestion.py b/contrib/metadata-ingestion/python/looker/dashboard_ingestion/looker_dashboard_ingestion.py deleted file mode 100644 index 22fc9d8718ea90..00000000000000 --- a/contrib/metadata-ingestion/python/looker/dashboard_ingestion/looker_dashboard_ingestion.py +++ /dev/null @@ -1,426 +0,0 @@ -#! /usr/bin/python -import time -import os -import json -import typing -from pprint import pprint -import looker_sdk -from looker_sdk.sdk.api31.models import Query, DashboardElement, LookWithQuery, Dashboard -from looker_sdk.error import SDKError - -from dataclasses import dataclass - -from confluent_kafka import avro -from confluent_kafka.avro import AvroProducer - -# Configuration -AVSC_PATH = "../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc" -KAFKA_TOPIC = 'MetadataChangeEvent_v4' - -# Set the following environmental variables to hit Looker's API -# LOOKERSDK_CLIENT_ID=YourClientID -# LOOKERSDK_CLIENT_SECRET=YourClientSecret -# LOOKERSDK_BASE_URL=https://company.looker.com:19999 -LOOKERSDK_BASE_URL = os.environ["LOOKERSDK_BASE_URL"] - -EXTRA_KAFKA_CONF = { - 'bootstrap.servers': 'localhost:9092', - 'schema.registry.url': 'http://localhost:8081' - # 'security.protocol': 'SSL', - # 'ssl.ca.location': '', - # 'ssl.key.location': '', - # 'ssl.certificate.location': '' -} - -# The datahub platform where looker views are stored, must be the same as VIEW_DATAHUB_PLATFORM in lookml_ingestion.py -VIEW_DATAHUB_PLATFORM = "looker_views" -# The datahub platform where looker dashboards will be stored -VISUALIZATION_DATAHUB_PLATFORM = "looker" - - -@dataclass -class LookerDashboardElement: - id: str - title: str - query_slug: str - looker_views: typing.List[str] - look_id: typing.Optional[str] - - @property - def url(self) -> str: - base_url = get_looker_base_url() - - # A dashboard element can use a look or just a raw query against an explore - if self.look_id is not None: - return base_url + "/looks/" + self.look_id - else: - return base_url + "/x/" + self.query_slug - - def get_urn_element_id(self): - # A dashboard element can use a look or just a raw query against an explore - return f"dashboard_elements.{self.id}" - - def get_view_urns(self) -> typing.List[str]: - return [f"urn:li:dataset:(urn:li:dataPlatform:{VIEW_DATAHUB_PLATFORM},{v},PROD)" for v in self.looker_views] - - -@dataclass -class LookerDashboard: - id: str - title: str - description: str - dashboard_elements: typing.List[LookerDashboardElement] - - @property - def url(self): - return get_looker_base_url() + "/dashboards/" + self.id - - def get_urn_dashboard_id(self): - return f"dashboards.{self.id}" - - -@dataclass -class DashboardKafkaEvents: - dashboard_mce: typing.Dict - chart_mces: typing.List[typing.Dict] - - def all_mces(self) -> typing.List[typing.Dict]: - return self.chart_mces + [self.dashboard_mce] - - -def get_looker_base_url(): - base_url = LOOKERSDK_BASE_URL.split("looker.com")[0] + "looker.com" - return base_url - - -def get_actor_and_sys_time(): - actor, sys_time = "urn:li:corpuser:analysts", int(time.time()) * 1000 - return actor, sys_time - - -class ProperDatahubEvents: - """ - This class generates events for "proper" datahub charts and dashboards - These events will not be visualized anywhere as of 12/11/2020 - """ - @staticmethod - def make_chart_mce(dashboard_element: LookerDashboardElement) -> typing.Dict: - actor, sys_time = get_actor_and_sys_time() - - owners = [{ - "owner": actor, - "type": "DEVELOPER" - }] - - return { - "auditHeader": None, - "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot", { - "urn": f"urn:li:chart:(looker,{dashboard_element.get_urn_element_id()})", - "aspects": [ - ("com.linkedin.pegasus2avro.dataset.ChartInfo", { - "title": dashboard_element.title, - "description": "", - "inputs": dashboard_element.get_view_urns(), - "url": f"", - "lastModified": {"created": {"time": sys_time, "actor": actor}} - }), - ("com.linkedin.pegasus2avro.common.Ownership", { - "owners": owners, - "lastModified": { - "time": sys_time, - "actor": actor - } - }) - ] - }), - "proposedDelta": None - } - - @staticmethod - def make_dashboard_mce(looker_dashboard: LookerDashboard) -> DashboardKafkaEvents: - actor, sys_time = get_actor_and_sys_time() - - owners = [{ - "owner": actor, - "type": "DEVELOPER" - }] - - chart_mces = [ProperDatahubEvents.make_chart_mce(element) for element in looker_dashboard.dashboard_elements] - - dashboard_mce = { - "auditHeader": None, - "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot", { - "urn": f"urn:li:dashboard:(looker,{looker_dashboard.get_urn_dashboard_id()},PROD)", - "aspects": [ - ("com.linkedin.pegasus2avro.dataset.DashboardInfo", { - "title": looker_dashboard.title, - "description": looker_dashboard.description, - "charts": [mce["proposedSnapshot"][1]["urn"] for mce in chart_mces], - "url": looker_dashboard.url, - "lastModified": {"created": {"time": sys_time, "actor": actor}} - }), - ("com.linkedin.pegasus2avro.common.Ownership", { - "owners": owners, - "lastModified": { - "time": sys_time, - "actor": actor - } - }) - ] - }), - "proposedDelta": None - } - - return DashboardKafkaEvents(dashboard_mce=dashboard_mce, chart_mces=chart_mces) - - -class WorkaroundDatahubEvents: - """ - This class generates events for "workaround" datahub charts and dashboards - This is so we can display end to end lineage without being blocked on datahub's support for dashboards and charts - - The approach is we generate "charts" and "dashboards" as just "datasets" in datahub under a new platform - We then link them together using "UpstreamLineage" just like any other dataset - """ - @staticmethod - def make_chart_mce(dashboard_element: LookerDashboardElement) -> typing.Dict: - actor, sys_time = get_actor_and_sys_time() - - owners = [{ - "owner": actor, - "type": "DEVELOPER" - }] - - upstreams = [{ - "auditStamp":{ - "time": sys_time, - "actor": actor - }, - "dataset": view_urn, - "type":"TRANSFORMED" - } for view_urn in dashboard_element.get_view_urns()] - - doc_elements = [{ - "url": dashboard_element.url, - "description": "Looker chart url", - "createStamp": { - "time": sys_time, - "actor": actor - } - }] - - return { - "auditHeader": None, - "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", { - "urn": f"urn:li:dataset:(urn:li:dataPlatform:{VISUALIZATION_DATAHUB_PLATFORM},{dashboard_element.get_urn_element_id()},PROD)", - "aspects": [ - ("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams": upstreams}), - ("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements": doc_elements}), - ("com.linkedin.pegasus2avro.dataset.DatasetProperties", {"description": dashboard_element.title, "customProperties": {}}), - ("com.linkedin.pegasus2avro.common.Ownership", { - "owners": owners, - "lastModified": { - "time": sys_time, - "actor": actor - } - }) - ] - }), - "proposedDelta": None - } - - @staticmethod - def make_dashboard_mce(looker_dashboard: LookerDashboard) -> DashboardKafkaEvents: - actor, sys_time = get_actor_and_sys_time() - - chart_mces = [WorkaroundDatahubEvents.make_chart_mce(element) for element in looker_dashboard.dashboard_elements] - - owners = [{ - "owner": actor, - "type": "DEVELOPER" - }] - - upstreams = [{ - "auditStamp":{ - "time": sys_time, - "actor": actor - }, - "dataset": chart_urn, - "type":"TRANSFORMED" - } for chart_urn in [mce["proposedSnapshot"][1]["urn"] for mce in chart_mces]] - - doc_elements = [{ - "url": looker_dashboard.url, - "description": "Looker dashboard url", - "createStamp": { - "time": sys_time, - "actor": actor - } - }] - - dashboard_mce = { - "auditHeader": None, - "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", { - "urn": f"urn:li:dataset:(urn:li:dataPlatform:{VISUALIZATION_DATAHUB_PLATFORM},{looker_dashboard.get_urn_dashboard_id()},PROD)", - "aspects": [ - ("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams": upstreams}), - ("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements": doc_elements}), - ("com.linkedin.pegasus2avro.dataset.DatasetProperties", {"description": looker_dashboard.title, "customProperties": {}}), - ("com.linkedin.pegasus2avro.common.Ownership", { - "owners": owners, - "lastModified": { - "time": sys_time, - "actor": actor - } - }) - ] - }), - "proposedDelta": None - } - - return DashboardKafkaEvents(dashboard_mce=dashboard_mce, chart_mces=chart_mces) - - -def delivery_report(err, msg): - """ Called once for each message produced to indicate delivery result. - Triggered by poll() or flush(). """ - if err is not None: - print('Message delivery failed: {}'.format(err)) - else: - print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition())) - - -def make_kafka_producer(extra_kafka_conf): - conf = { - "on_delivery": delivery_report, - **extra_kafka_conf - } - - key_schema = avro.loads('{"type": "string"}') - record_schema = avro.load(AVSC_PATH) - producer = AvroProducer(conf, default_key_schema=key_schema, default_value_schema=record_schema) - return producer - - -def _extract_view_from_field(field: str) -> str: - assert field.count(".") == 1, f"Error: A field must be prefixed by a view name, field is: {field}" - view_name = field.split(".")[0] - return view_name - - -def get_views_from_query(query: Query) -> typing.List[str]: - all_views = set() - - # query.dynamic_fields can contain: - # - looker table calculations: https://docs.looker.com/exploring-data/using-table-calculations - # - looker custom measures: https://docs.looker.com/de/exploring-data/adding-fields/custom-measure - # - looker custom dimensions: https://docs.looker.com/exploring-data/adding-fields/custom-measure#creating_a_custom_dimension_using_a_looker_expression - dynamic_fields = json.loads(query.dynamic_fields if query.dynamic_fields is not None else '[]') - custom_field_to_underlying_field = {} - for field in dynamic_fields: - # Table calculations can only reference fields used in the fields section, so this will always be a subset of of the query.fields - if "table_calculation" in field: - continue - # Looker custom measures can reference fields in arbitrary views, so this needs to be parsed to find the underlying view field the custom measure is based on - if "measure" in field: - measure = field["measure"] - based_on = field["based_on"] - custom_field_to_underlying_field[measure] = based_on - - # Looker custom dimensions can reference fields in arbitrary views, so this needs to be parsed to find the underlying view field the custom measure is based on - # However, unlike custom measures custom dimensions can be defined using an arbitrary expression - # We are not going to support parsing arbitrary Looker expressions here, so going to ignore these fields for now - # TODO: support parsing arbitrary looker expressions - if "dimension" in field: - dimension = field["dimension"] - expression = field["expression"] - custom_field_to_underlying_field[dimension] = None - - # A query uses fields defined in views, find the views those fields use - fields: typing.Sequence[str] = query.fields if query.fields is not None else [] - for field in fields: - # If the field is a custom field, look up the field it is based on - field_name = custom_field_to_underlying_field[field] if field in custom_field_to_underlying_field else field - if field_name is None: - continue - view_name = _extract_view_from_field(field_name) - all_views.add(view_name) - - # A query uses fields for filtering and those fields are defined in views, find the views those fields use - filters: typing.MutableMapping[str, typing.Any] = query.filters if query.filters is not None else {} - for field in filters.keys(): - # If the field is a custom field, look up the field it is based on - field_name = custom_field_to_underlying_field[field] if field in custom_field_to_underlying_field else field - if field_name is None: - continue - view_name = _extract_view_from_field(field_name) - all_views.add(view_name) - - return list(all_views) - - -def get_views_from_look(look: LookWithQuery): - return get_views_from_query(look.query) - - -def get_looker_dashboard_element(element: DashboardElement)-> typing.Optional[LookerDashboardElement]: - # Dashboard elements can use raw queries against explores - if element.query is not None: - views = get_views_from_query(element.query) - return LookerDashboardElement(id=element.id, title=element.title, look_id=None, query_slug=element.query.slug, looker_views=views) - - # Dashboard elements can *alternatively* link to an existing look - if element.look is not None: - views = get_views_from_look(element.look) - return LookerDashboardElement(id=element.id, title=element.title, look_id=element.look_id, query_slug=element.look.query.slug, looker_views=views) - - # This occurs for "text" dashboard elements that just contain static text (ie: no queries) - # There is not much meaningful info to extract from these elements, so ignore them - return None - - -def get_looker_dashboard(dashboard: Dashboard) -> LookerDashboard: - dashboard_elements: typing.List[LookerDashboardElement] = [] - for element in dashboard.dashboard_elements: - looker_dashboard_element = get_looker_dashboard_element(element) - if looker_dashboard_element is not None: - dashboard_elements.append(looker_dashboard_element) - - looker_dashboard = LookerDashboard(id=dashboard.id, title=dashboard.title, description=dashboard.description, dashboard_elements=dashboard_elements) - return looker_dashboard - - -# Perform IO in main -def main(): - kafka_producer = make_kafka_producer(EXTRA_KAFKA_CONF) - sdk = looker_sdk.init31() - dashboard_ids = [dashboard_base.id for dashboard_base in sdk.all_dashboards(fields="id")] - - looker_dashboards = [] - for dashboard_id in dashboard_ids: - try: - fields = ["id", "title", "dashboard_elements", "dashboard_filters"] - dashboard_object = sdk.dashboard(dashboard_id=dashboard_id, fields=",".join(fields)) - except SDKError as e: - # A looker dashboard could be deleted in between the list and the get - print(f"Skipping dashboard with dashboard_id: {dashboard_id}") - print(e) - continue - - looker_dashboard = get_looker_dashboard(dashboard_object) - looker_dashboards.append(looker_dashboard) - pprint(looker_dashboard) - - for looker_dashboard in looker_dashboards: - workaround_dashboard_kafka_events = WorkaroundDatahubEvents.make_dashboard_mce(looker_dashboard) - # Hard to test these events since datahub does not have a UI, for now disable sending them - # proper_dashboard_kafka_events = ProperDatahubEvents.make_dashboard_mce(looker_dashboard) - - for mce in workaround_dashboard_kafka_events.all_mces(): - print(mce) - kafka_producer.produce(topic=KAFKA_TOPIC, key=mce['proposedSnapshot'][1]['urn'], value=mce) - kafka_producer.flush() - - -if __name__ == "__main__": - main() diff --git a/contrib/metadata-ingestion/python/looker/dashboard_ingestion/requirements.txt b/contrib/metadata-ingestion/python/looker/dashboard_ingestion/requirements.txt deleted file mode 100644 index 14c3c7828458b9..00000000000000 --- a/contrib/metadata-ingestion/python/looker/dashboard_ingestion/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -avro-python3==1.8.2 -confluent-kafka[avro]==1.4.0 -PyYAML==5.4.1 -looker-sdk==0.1.3b20 \ No newline at end of file diff --git a/contrib/metadata-ingestion/python/looker/lookml_ingestion/README.md b/contrib/metadata-ingestion/python/looker/lookml_ingestion/README.md deleted file mode 100644 index 994c123f5f1214..00000000000000 --- a/contrib/metadata-ingestion/python/looker/lookml_ingestion/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## lookml_ingestion.py -This script ingests Looker view metadata from lookml into datahub. Looker views are essentially like database views that can be either materialized or ephemeral, so we treat them as you would any other dataset in datahub. - -Underneath the hood, this script uses the `lkml` python parsing library to parse lkml files and so it comes with all the limitations of that underlying parser. - -Roughly how the script works: -- Point the script at a directory on the filesystem, finds all files named `*.model.lkml` in any level of nesting -- Finds the viewfile includes in the model file, this indicates that the viewfile is a part of that model (and a model has a single SQL connection associated with it). Does not handle a model importing a view file but *not* using the view in the model since that would require parsing explore blocks and adds complexity. -- For each viewfile in the model, parses the view files. For each view in the viewfile, resolve the sql table name for the view: - - We do not support parsing derived tables using a `sql:` block, this would require parsing SQL to understand dependencies. We only support views using `sql_table_name`. In the future, could support limited SQL parsing for limited SQL dialects. - - We support views using the `extends` keyword: https://docs.looker.com/reference/view-params/extends-for-view This is surprisingly painful because views can extend other views in other files. We do this inefficiently right now. - - We do not support views using `refinements`. SpotHero does not use refinements right now, so we had no need to implement it: https://docs.looker.com/data-modeling/learning-lookml/refinements -- After binding views to models and finding the sql table name associated with the views, we generate the MCE events into a separate looker platform in datahub since they are not "real" tables but "virtual" looker tables - -## Steps -- Use a version of python >= 3.7 -- Make a virtual environment -- pip install -r requirements.txt -- Set env var: LOOKER_DIRECTORY to the root path of lkml on your filesystem -- Modify EXTRA_KAFKA_CONF section of script to point to datahub \ No newline at end of file diff --git a/contrib/metadata-ingestion/python/looker/lookml_ingestion/lookml_ingestion.py b/contrib/metadata-ingestion/python/looker/lookml_ingestion/lookml_ingestion.py deleted file mode 100644 index a73583e1eb0da7..00000000000000 --- a/contrib/metadata-ingestion/python/looker/lookml_ingestion/lookml_ingestion.py +++ /dev/null @@ -1,343 +0,0 @@ -import lkml -import glob -import time -import typing -import os -import re - -from confluent_kafka import avro -from confluent_kafka.avro import AvroProducer - -from dataclasses import dataclass, replace - -from sql_metadata import get_query_tables - -# Configuration -AVSC_PATH = "../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeEvent.avsc" -KAFKA_TOPIC = 'MetadataChangeEvent_v4' - -# LOOKER_DIRECTORY = "./test_lookml" -LOOKER_DIRECTORY = os.environ["LOOKER_DIRECTORY"] -LOOKER_DIRECTORY = os.path.abspath(LOOKER_DIRECTORY) - -EXTRA_KAFKA_CONF = { - 'bootstrap.servers': 'localhost:9092', - 'schema.registry.url': 'http://localhost:8081' - # 'security.protocol': 'SSL', - # 'ssl.ca.location': '', - # 'ssl.key.location': '', - # 'ssl.certificate.location': '' -} - -# The datahub platform where looker views are stored -LOOKER_VIEW_PLATFORM = "looker_views" - -class LookerViewFileLoader: - """ - Loads the looker viewfile at a :path and caches the LookerViewFile in memory - This is to avoid reloading the same file off of disk many times during the recursive include resolution process - """ - - def __init__(self): - self.viewfile_cache = {} - - def _load_viewfile(self, path: str) -> typing.Optional["LookerViewFile"]: - if path in self.viewfile_cache: - return self.viewfile_cache[path] - - try: - with open(path, "r") as file: - parsed = lkml.load(file) - looker_viewfile = LookerViewFile.from_looker_dict(path, parsed) - self.viewfile_cache[path] = looker_viewfile - return looker_viewfile - except Exception as e: - print(e) - print(f"Error processing view file {path}. Skipping it") - - def load_viewfile(self, path: str, connection: str): - viewfile = self._load_viewfile(path) - if viewfile is None: - return None - - return replace(viewfile, connection=connection) - - - -@dataclass -class LookerModel: - connection: str - includes: typing.List[str] - resolved_includes: typing.List[str] - - @staticmethod - def from_looker_dict(looker_model_dict): - connection = looker_model_dict["connection"] - includes = looker_model_dict["includes"] - resolved_includes = LookerModel.resolve_includes(includes) - - return LookerModel(connection=connection, includes=includes, resolved_includes=resolved_includes) - - @staticmethod - def resolve_includes(includes) -> typing.List[str]: - resolved = [] - for inc in includes: - # Massage the looker include into a valid glob wildcard expression - glob_expr = f"{LOOKER_DIRECTORY}/{inc}" - outputs = glob.glob(glob_expr) - resolved.extend(outputs) - return resolved - - -@dataclass -class LookerViewFile: - absolute_file_path: str - connection: typing.Optional[str] - includes: typing.List[str] - resolved_includes: typing.List[str] - views: typing.List[typing.Dict] - - @staticmethod - def from_looker_dict(absolute_file_path, looker_view_file_dict): - includes = looker_view_file_dict.get("includes", []) - resolved_includes = LookerModel.resolve_includes(includes) - views = looker_view_file_dict.get("views", []) - - return LookerViewFile(absolute_file_path=absolute_file_path, connection=None, includes=includes, resolved_includes=resolved_includes, views=views) - -@dataclass -class LookerView: - absolute_file_path: str - connection: str - view_name: str - sql_table_names: typing.List[str] - - def get_relative_file_path(self): - if LOOKER_DIRECTORY in self.absolute_file_path: - return self.absolute_file_path.replace(LOOKER_DIRECTORY, '').lstrip('/') - else: - raise Exception(f"Found a looker view with name: {view_name} at path: {absolute_file_path} not underneath the base LOOKER_DIRECTORY: {LOOKER_DIRECTORY}. This should not happen") - - @staticmethod - def from_looker_dict(looker_view, connection: str, looker_viewfile: LookerViewFile, looker_viewfile_loader: LookerViewFileLoader) -> typing.Optional["LookerView"]: - view_name = looker_view["name"] - sql_table_name = looker_view.get("sql_table_name", None) - # Some sql_table_name fields contain quotes like: optimizely."group", just remove the quotes - sql_table_name = sql_table_name.replace('"', '') if sql_table_name is not None else None - derived_table = looker_view.get("derived_table", None) - - # Parse SQL from derived tables to extract dependencies - if derived_table is not None and 'sql' in derived_table: - # Get the list of tables in the query - sql_tables: typing.List[str] = get_query_tables(derived_table['sql']) - - # Remove temporary tables from WITH statements - sql_table_names = [t for t in sql_tables if not re.search(f'WITH(.*,)?\s+{t}(\s*\([\w\s,]+\))?\s+AS\s+\(', derived_table['sql'], re.IGNORECASE|re.DOTALL)] - - # Remove quotes from tables - sql_table_names = [t.replace('"', '') for t in sql_table_names] - - return LookerView(absolute_file_path=looker_viewfile.absolute_file_path, connection=connection, view_name=view_name, sql_table_names=sql_table_names) - - # There is a single dependency in the view, on the sql_table_name - if sql_table_name is not None: - return LookerView(absolute_file_path=looker_viewfile.absolute_file_path, connection=connection, view_name=view_name, sql_table_names=[sql_table_name]) - - # The sql_table_name might be defined in another view and this view is extending that view, try to find it - else: - extends = looker_view.get("extends", []) - if len(extends) == 0: - # The view is malformed, the view is not a derived table, does not contain a sql_table_name or an extends - print(f"Skipping malformed with view_name: {view_name}. View should have a sql_table_name if it is not a derived table") - return None - - extends_to_looker_view = [] - - # The base view could live in the same file - for raw_view in looker_viewfile.views: - raw_view_name = raw_view["name"] - # Make sure to skip loading view we are currently trying to resolve - if raw_view_name != view_name: - maybe_looker_view = LookerView.from_looker_dict(raw_view, connection, looker_viewfile, looker_viewfile_loader) - if maybe_looker_view is not None and maybe_looker_view.view_name in extends: - extends_to_looker_view.append(maybe_looker_view) - - # Or it could live in one of the included files, we do not know which file the base view lives in, try them all! - for include in looker_viewfile.resolved_includes: - looker_viewfile = looker_viewfile_loader.load_viewfile(include, connection) - if looker_viewfile is not None: - for view in looker_viewfile.views: - maybe_looker_view = LookerView.from_looker_dict(view, connection, looker_viewfile, looker_viewfile_loader) - if maybe_looker_view is None: - continue - - if maybe_looker_view is not None and maybe_looker_view.view_name in extends: - extends_to_looker_view.append(maybe_looker_view) - - if len(extends_to_looker_view) != 1: - print(f"Skipping malformed view with view_name: {view_name}. View should have a single view in a view inheritance chain with a sql_table_name") - return None - - output_looker_view = LookerView(absolute_file_path=looker_viewfile.absolute_file_path, connection=connection, view_name=view_name, sql_table_names=extends_to_looker_view[0].sql_table_names) - return output_looker_view - - - - -def get_platform_and_table(view_name: str, connection: str, sql_table_name: str): - """ - This will depend on what database connections you use in Looker - For SpotHero, we had two database connections in Looker: "redshift_test" (a redshift database) and "presto" (a presto database) - Presto supports querying across multiple catalogs, so we infer which underlying database presto is using based on the presto catalog name - For SpotHero, we have 3 catalogs in presto: "redshift", "hive", and "hive_emr" - """ - if connection == "redshift_test": - platform = "redshift" - table_name = sql_table_name - return platform, table_name - - elif connection == "presto": - parts = sql_table_name.split(".") - catalog = parts[0] - - if catalog == "hive": - platform = "hive" - elif catalog == "hive_emr": - platform = "hive_emr" - elif catalog == "redshift": - platform = "redshift" - else: - # Looker lets you exclude a catalog and use a configured default, the default we have configured is to use hive_emr - if sql_table_name.count(".") != 1: - raise Exception("Unknown catalog for sql_table_name: {sql_table_name} for view_name: {view_name}") - - platform = "hive_emr" - return platform, sql_table_name - - table_name = ".".join(parts[1::]) - return platform, table_name - else: - raise Exception(f"Could not find a platform for looker view with connection: {connection}") - - -def construct_datalineage_urn(view_name: str, connection: str, sql_table_name: str): - platform, table_name = get_platform_and_table(view_name, connection, sql_table_name) - return f"urn:li:dataset:(urn:li:dataPlatform:{platform},{table_name},PROD)" - -def construct_data_urn(looker_view: LookerView): - return f"urn:li:dataset:(urn:li:dataPlatform:{LOOKER_VIEW_PLATFORM},{looker_view.view_name},PROD)" - - - -def build_dataset_mce(looker_view: LookerView): - """ - Creates MetadataChangeEvent for the dataset, creating upstream lineage links - """ - actor, sys_time = "urn:li:corpuser:etl", int(time.time()) * 1000 - - upstreams = [{ - "auditStamp":{ - "time": sys_time, - "actor":actor - }, - "dataset": construct_datalineage_urn(looker_view.view_name, looker_view.connection, sql_table_name), - "type":"TRANSFORMED" - } for sql_table_name in looker_view.sql_table_names] - - - doc_elements = [{ - "url":f"https://github.com/spothero/internal-looker-repo/blob/master/{looker_view.get_relative_file_path()}", - "description":"Github looker view definition", - "createStamp":{ - "time": sys_time, - "actor": actor - } - }] - - owners = [{ - "owner": f"urn:li:corpuser:analysts", - "type": "DEVELOPER" - }] - - return { - "auditHeader": None, - "proposedSnapshot":("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", { - "urn": construct_data_urn(looker_view), - "aspects": [ - ("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams": upstreams}), - ("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements": doc_elements}), - ("com.linkedin.pegasus2avro.common.Ownership", { - "owners": owners, - "lastModified":{ - "time": sys_time, - "actor": actor - } - }) - ] - }), - "proposedDelta": None - } - - -def delivery_report(err, msg): - """ Called once for each message produced to indicate delivery result. - Triggered by poll() or flush(). """ - if err is not None: - print('Message delivery failed: {}'.format(err)) - else: - print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition())) - - -def make_kafka_producer(extra_kafka_conf): - conf = { - "on_delivery": delivery_report, - **extra_kafka_conf - } - - key_schema = avro.loads('{"type": "string"}') - record_schema = avro.load(AVSC_PATH) - producer = AvroProducer(conf, default_key_schema=key_schema, default_value_schema=record_schema) - return producer - - -def main(): - kafka_producer = make_kafka_producer(EXTRA_KAFKA_CONF) - viewfile_loader = LookerViewFileLoader() - - looker_models = [] - all_views = [] - - model_files = sorted(f for f in glob.glob(f"{LOOKER_DIRECTORY}/**/*.model.lkml", recursive=True)) - for f in model_files: - try: - with open(f, 'r') as file: - parsed = lkml.load(file) - looker_model = LookerModel.from_looker_dict(parsed) - looker_models.append(looker_model) - except Exception as e: - print(e) - print(f"Error processing model file {f}. Skipping it") - - - - for model in looker_models: - for include in model.resolved_includes: - looker_viewfile = viewfile_loader.load_viewfile(include, model.connection) - if looker_viewfile is not None: - for raw_view in looker_viewfile.views: - maybe_looker_view = LookerView.from_looker_dict(raw_view, model.connection, looker_viewfile, viewfile_loader) - if maybe_looker_view: - all_views.append(maybe_looker_view) - - - for view in all_views: - MCE = build_dataset_mce(view) - print(view) - print(MCE) - kafka_producer.produce(topic=KAFKA_TOPIC, key=MCE['proposedSnapshot'][1]['urn'], value=MCE) - kafka_producer.flush() - - - -if __name__ == "__main__": - main() diff --git a/contrib/metadata-ingestion/python/looker/lookml_ingestion/requirements.txt b/contrib/metadata-ingestion/python/looker/lookml_ingestion/requirements.txt deleted file mode 100644 index 8f9fd5d9999906..00000000000000 --- a/contrib/metadata-ingestion/python/looker/lookml_ingestion/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -lkml==1.1.0 -avro-python3==1.8.2 -confluent-kafka[avro]==1.4.0 -sql-metadata==1.12.0 diff --git a/contrib/nix/README.md b/contrib/nix/README.md deleted file mode 100644 index d768f08edb0bd7..00000000000000 --- a/contrib/nix/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Nix sandbox for datahub - - -## Introduction -database is not suitable for virtualization for it's io performance. - -so we use simple nix package tool to install package and setup service on physical machine. - -we declare it, then it works. see [sandbox.nix] file for details. - -it install software on /nix directory, and run service on launchpad(darwin) and systemd(linux). - -NOTE: for linux, ensure 'systemd --user' process running. - - -## Roadmap - -- [X] support mac and linux -- [ ] add environment check script -- [ ] add datahub nix package -- [ ] add datahub[gms, frontend, pipeline] service module -- [ ] add nixops distributed deploy - - -## Quickstart -1. install nix and channel - -``` - sudo install -d -m755 -o $(id -u) -g $(id -g) /nix - curl https://nixos.org/nix/install | sh - - nix-channel --add https://nixos.org/channels/nixos-20.03 nixpkgs - nix-channel --update nixpkgs -``` - -2. install home-manager - -``` - nix-channel --add https://github.com/clojurians-org/home-manager/archive/v1.0.0.tar.gz home-manager - nix-channel --update home-manager - NIX_PATH=~/.nix-defexpr/channels nix-shell '' -A install -``` - -3. setup environment, and well done! -``` - NIX_PATH=~/.nix-defexpr/channels home-manager -f sandbox.nix switch -``` - -## Client connect -``` -mysql => mysql -u root -S /nix/var/run/mysqld.sock -postgresql => psql -h /nix/var/run postgres -elasticsearch => curl http://localhost:9200 -neo4j => cypher-shell -uneo4j -pneo4j -zookeeper => zkCli.sh -kafka => kafka-topics.sh --bootstrap-server localhost:9092 --list -confluent schema-registry => curl http://localhost:8081 - -``` - -## Environemnt Check - -you only need install nix to run it! - -``` -nix-shell datahub-check.nix -A gms -``` diff --git a/contrib/nix/datahub-check.nix b/contrib/nix/datahub-check.nix deleted file mode 100644 index aee544283f8e11..00000000000000 --- a/contrib/nix/datahub-check.nix +++ /dev/null @@ -1,99 +0,0 @@ -{ pkgs ? import {} }: - -with pkgs ; -let - datahub = import ./datahub-config.nix ; - build-prompt = '' - echo This derivation is not buildable, instead run it using nix-shell. - exit 1 - '' ; - parse-uri = uri : - let - uriSchemaSplit = builtins.split "://" uri ; - schema = builtins.head uriSchemaSplit ; - uriNoSchema = lib.last uriSchemaSplit ; - - uriPathSplit = builtins.split "/" uriNoSchema ; - hostPort = builtins.head uriPathSplit ; - path = lib.optionalString (builtins.length uriPathSplit > 1) (lib.last uriPathSplit) ; - - hostPortSplit = builtins.split ":" hostPort ; - host = builtins.head hostPortSplit ; - port = lib.last hostPortSplit ; - - in { inherit schema host port path ; } ; - gms = - let - gms-conf = datahub.services.linkedin-datahub-gms ; - jdbc-uri = parse-uri gms-conf.sandbox.jdbc.uri ; - elasticsearch-uri = parse-uri (builtins.head gms-conf.sandbox.elasticsearch.uris) ; - neo4j-uri = parse-uri gms-conf.sandbox.neo4j.uri ; - kafka-uri = parse-uri (builtins.head gms-conf.sandbox.kafka.uris) ; - schema-registry-uri = parse-uri (builtins.head gms-conf.sandbox.schema-registry.uris) ; - gms-uri = parse-uri gms-conf.listener ; - - check-port = name : uri : '' - echo " [${name}] checking port..." - ${netcat-gnu}/bin/nc -z ${uri.host} ${uri.port} - if [ $? != 0 ]; then echo " [${name}] !ERROR: can not connec to ${uri.host}:${uri.port}" && exit 1; fi - '' ; - check-jdbc-user = '' - # echo " [jdbc] checking username and password..." - '' ; - check-jdbc-table = '' - # echo " [jdbc] checking [metadata_aspect] table..." - '' ; - check-elasticsearch-index = '' - # echo " [elasticsearch] checking [corpuserinfodocument, datasetdocument] indices ..." - '' ; - check-neo4j-user = '' - # echo " [neo4j] checking user and password..." - '' ; - check-kafka-topic = '' - # echo " [kafka] checking [MetadataChangeEvent, MetadataAuditEvent] indices..." - '' ; - in - stdenv.mkDerivation { - name = "gms-check" ; - - buildInputs = [ netcat-gnu ] ; - - preferLocalBuild = true ; - buildCommand = build-prompt ; - - shellHookOnly = true; - shellHook = '' - echo "******** checking sandbox.jdbc " - ${check-port "jdbc" jdbc-uri} - ${check-jdbc-user } - ${check-jdbc-table } - - echo "******** checking sandbox.elasticsearch " - ${check-port "elasticsearch" elasticsearch-uri} - ${check-elasticsearch-index} - - echo "******** checking sandbox.neo4j " - ${check-port "neo4j" neo4j-uri} - ${check-neo4j-user } - - echo "******** checking sandbox.kafka " - ${check-port "kafka" kafka-uri} - ${check-kafka-topic } - - echo "******** checking sandbox.schema-registry " - ${check-port "schema-registry" schema-registry-uri} - - echo "******** checking gms " - ${check-port "gms" gms-uri} - exit 0 - '' ; - } ; - frontend = - let - frontend-conf = ddatahub.services.linkedin-datahub-frontend ; - in {} ; - pipeline = - let - pipeline-conf = ddatahub.services.linkedin-datahub-pipeline ; - in {} ; -in { inherit gms frontend pipeline;} diff --git a/contrib/nix/datahub-config.nix b/contrib/nix/datahub-config.nix deleted file mode 100644 index 25a212b807aa7d..00000000000000 --- a/contrib/nix/datahub-config.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ - services.linkedin-datahub-gms = { - enable = true; - sandbox = { - jdbc.uri = "jdbc:postgresql://localhost:5432/datahub" ; - jdbc.username = "datahub" ; - jdbc.password = "datahub" ; - elasticsearch.uris = [ "http://localhost:9200" ] ; - neo4j.uri = "bolt://localhost:7687" ; - neo4j.username = "neo4j" ; - neo4j.password = "datahub" ; - kafka.uris = [ "PLAINTEXT://localhost:9092" ] ; - schema-registry.uris = [ "http://localhost:8081" ] ; - } ; - listener = "http://localhost:8080" ; - } ; - - services.linkedin-datahub-frontend = { - enable = true ; - listener = "http://localhost:9001" ; - linkedin-datahub-gms.uri = "http://localhost:8080" ; - } ; - services.linkedin-datahub-pipeline = { - enable = true ; - linkedin-datahub-gms.uri = "http://localhost:8080" ; - sandbox = { - kafka.uris = [ "PLAINTEXT://localhost:9092" ] ; - schema-registry.uris = [ "http://localhost:8081" ] ; - } ; - } ; -} diff --git a/contrib/nix/sandbox.nix b/contrib/nix/sandbox.nix deleted file mode 100644 index a6f2b95529c8ba..00000000000000 --- a/contrib/nix/sandbox.nix +++ /dev/null @@ -1,80 +0,0 @@ -{ config, pkgs, ... }: - -{ - # Let Home Manager install and manage itself. - programs.home-manager.enable = true; - - # This value determines the Home Manager release that your - # configuration is compatible with. This helps avoid breakage - # when a new Home Manager release introduces backwards - # incompatible changes. - # - # You can update Home Manager without changing this value. See - # the Home Manager release notes for a list of state version - # changes in each release. - home.stateVersion = "19.09"; - - - environment.systemPackages = [ - pkgs.gradle - - pkgs.postgresql_11 - pkgs.mysql57 - pkgs.elasticsearch - pkgs.neo4j - pkgs.zookeeper - pkgs.apacheKafka - pkgs.confluent-platform - pkgs.kafkacat - pkgs.neo4j - ]; - - services.postgresql = { - enable = true ; - package = pkgs.postgresql_11 ; - dataDir = "/opt/nix-module/data/postgresql" ; - } ; - - services.mysql = { - enable = true ; - # package = pkgs.mysql80 ; - package = pkgs.mysql57 ; - dataDir = "/opt/nix-module/data/mysql" ; - } ; - - services.elasticsearch = { - enable = true ; - # package = pkgs.elasticsearch7 ; - package = pkgs.elasticsearch ; - dataDir = "/opt/nix-module/data/elasticsearch" ; - } ; - - services.neo4j = { - enable = true ; - package = pkgs.neo4j ; - directories.home = "/opt/nix-module/data/neo4j" ; - } ; - - services.zookeeper = { - enable = true ; - package = pkgs.zookeeper ; - dataDir = "/opt/nix-module/data/zookeeper" ; - } ; - - services.apache-kafka = { - enable = true ; - package = pkgs.apacheKafka ; - logDirs = [ "/opt/nix-module/data/kafka" ] ; - zookeeper = "localhost:2181" ; - extraProperties = '' - offsets.topic.replication.factor = 1 - zookeeper.session.timeout.ms = 600000 - '' ; - } ; - - services.confluent-schema-registry = { - enable = true ; - package = pkgs.confluent-platform ; - kafkas = [ "PLAINTEXT://localhost:9092" ] ; - } ; -} diff --git a/datahub-frontend/README.md b/datahub-frontend/README.md index 44259335b65ad0..0bfb796562eeaa 100644 --- a/datahub-frontend/README.md +++ b/datahub-frontend/README.md @@ -62,7 +62,7 @@ WHZ-Authentication { ### Authentication in React The React app supports both JAAS as described above and separately OIDC authentication. To learn about configuring OIDC for React, -see the [OIDC in React](../docs/how/auth/sso/configure-oidc-react.md) document. +see the [OIDC in React](../docs/authentication/guides/sso/configure-oidc-react.md) document. ### API Debugging diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 1bf2de6ada7c48..0773d95b45cf58 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -11,7 +11,9 @@ import com.linkedin.metadata.restli.DefaultRestliClientFactory; import com.linkedin.util.Configuration; import com.datahub.authentication.Authentication; +import java.nio.charset.StandardCharsets; import java.util.Collections; +import org.apache.commons.codec.digest.DigestUtils; import org.pac4j.core.client.Client; import org.pac4j.core.client.Clients; import org.pac4j.core.config.Config; @@ -19,7 +21,9 @@ import org.pac4j.play.LogoutController; import org.pac4j.play.http.PlayHttpActionAdapter; import org.pac4j.play.store.PlayCacheSessionStore; +import org.pac4j.play.store.PlayCookieSessionStore; import org.pac4j.play.store.PlaySessionStore; +import org.pac4j.play.store.ShiroAesDataEncrypter; import play.Environment; import java.util.ArrayList; @@ -49,6 +53,8 @@ public class AuthModule extends AbstractModule { * We hash this value (SHA1), then take the first 16 bytes as the AES key. */ private static final String PAC4J_AES_KEY_BASE_CONF = "play.http.secret.key"; + private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider"; + private final com.typesafe.config.Config _configs; public AuthModule(final Environment environment, final com.typesafe.config.Config configs) { @@ -57,10 +63,38 @@ public AuthModule(final Environment environment, final com.typesafe.config.Confi @Override protected void configure() { - - final PlayCacheSessionStore playCacheSessionStore = new PlayCacheSessionStore(getProvider(SyncCacheApi.class)); - bind(SessionStore.class).toInstance(playCacheSessionStore); - bind(PlaySessionStore.class).toInstance(playCacheSessionStore); + /** + * In Pac4J, you are given the option to store the profiles of authenticated users in either + * (i) PlayCacheSessionStore - saves your data in the Play cache or + * (ii) PlayCookieSessionStore saves your data in the Play session cookie + * However there is problem (https://github.com/datahub-project/datahub/issues/4448) observed when storing the Pac4j profile in cookie. + * Whenever the profile returned by Pac4j is greater than 4096 characters, the response will be rejected by the browser. + * Default to PlayCacheCookieStore so that datahub-frontend container remains as a stateless service + */ + String sessionStoreProvider = _configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF); + + if (sessionStoreProvider.equals("PlayCacheSessionStore")) { + final PlayCacheSessionStore playCacheSessionStore = new PlayCacheSessionStore(getProvider(SyncCacheApi.class)); + bind(SessionStore.class).toInstance(playCacheSessionStore); + bind(PlaySessionStore.class).toInstance(playCacheSessionStore); + } else { + PlayCookieSessionStore playCacheCookieStore; + try { + // To generate a valid encryption key from an input value, we first + // hash the input to generate a fixed-length string. Then, we convert + // it to hex and slice the first 16 bytes, because AES key length must strictly + // have a specific length. + final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF); + final String aesKeyHash = DigestUtils.sha1Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8)); + final String aesEncryptionKey = aesKeyHash.substring(0, 16); + playCacheCookieStore = new PlayCookieSessionStore( + new ShiroAesDataEncrypter(aesEncryptionKey)); + } catch (Exception e) { + throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e); + } + bind(SessionStore.class).toInstance(playCacheCookieStore); + bind(PlaySessionStore.class).toInstance(playCacheCookieStore); + } try { bind(SsoCallbackController.class).toConstructor(SsoCallbackController.class.getConstructor( diff --git a/datahub-frontend/app/auth/AuthUtils.java b/datahub-frontend/app/auth/AuthUtils.java index 9329012966a4ae..a32419ace115ed 100644 --- a/datahub-frontend/app/auth/AuthUtils.java +++ b/datahub-frontend/app/auth/AuthUtils.java @@ -47,6 +47,11 @@ public class AuthUtils { public static final String PASSWORD = "password"; public static final String ACTOR = "actor"; public static final String ACCESS_TOKEN = "token"; + public static final String FULL_NAME = "fullName"; + public static final String EMAIL = "email"; + public static final String TITLE = "title"; + public static final String INVITE_TOKEN = "inviteToken"; + public static final String RESET_TOKEN = "resetToken"; /** * Determines whether the inbound request should be forward to downstream Metadata Service. Today, this simply diff --git a/datahub-frontend/app/auth/JAASConfigs.java b/datahub-frontend/app/auth/JAASConfigs.java index 53f98ca2b08a48..f39c20aceb6f9b 100644 --- a/datahub-frontend/app/auth/JAASConfigs.java +++ b/datahub-frontend/app/auth/JAASConfigs.java @@ -11,9 +11,7 @@ public class JAASConfigs { private Boolean _isEnabled = true; public JAASConfigs(final com.typesafe.config.Config configs) { - if (configs.hasPath(JAAS_ENABLED_CONFIG_PATH) - && Boolean.FALSE.equals( - Boolean.parseBoolean(configs.getValue(JAAS_ENABLED_CONFIG_PATH).toString()))) { + if (configs.hasPath(JAAS_ENABLED_CONFIG_PATH) && !configs.getBoolean(JAAS_ENABLED_CONFIG_PATH)) { _isEnabled = false; } } diff --git a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java new file mode 100644 index 00000000000000..db17313d67f9a4 --- /dev/null +++ b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java @@ -0,0 +1,23 @@ +package auth; + +/** + * Currently, this config enables or disable native user authentication. + */ +public class NativeAuthenticationConfigs { + + public static final String NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH = "auth.native.enabled"; + + private Boolean _isEnabled = true; + + public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) { + if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH) + && Boolean.FALSE.equals( + Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString()))) { + _isEnabled = false; + } + } + + public boolean isNativeAuthenticationEnabled() { + return _isEnabled; + } +} diff --git a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java new file mode 100644 index 00000000000000..723601d7f3fb18 --- /dev/null +++ b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java @@ -0,0 +1,50 @@ +package auth.sso.oidc; + +import java.util.Map.Entry; + +import org.pac4j.core.authorization.generator.AuthorizationGenerator; +import org.pac4j.core.context.WebContext; +import org.pac4j.core.profile.AttributeLocation; +import org.pac4j.core.profile.CommonProfile; +import org.pac4j.core.profile.definition.ProfileDefinition; +import org.pac4j.oidc.profile.OidcProfile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.nimbusds.jwt.JWT; +import com.nimbusds.jwt.JWTParser; + +public class OidcAuthorizationGenerator implements AuthorizationGenerator { + + private static final Logger logger = LoggerFactory.getLogger(OidcAuthorizationGenerator.class); + + private final ProfileDefinition profileDef; + + private final OidcConfigs oidcConfigs; + + public OidcAuthorizationGenerator(final ProfileDefinition profileDef, final OidcConfigs oidcConfigs) { + this.profileDef = profileDef; + this.oidcConfigs = oidcConfigs; + } + + @Override + public CommonProfile generate(WebContext context, CommonProfile profile) { + if (oidcConfigs.getExtractJwtAccessTokenClaims().orElse(false)) { + try { + final JWT jwt = JWTParser.parse(((OidcProfile) profile).getAccessToken().getValue()); + + for (final Entry entry : jwt.getJWTClaimsSet().getClaims().entrySet()) { + final String claimName = entry.getKey(); + if (profile.getAttribute(claimName) == null) { + profileDef.convertAndAdd(profile, AttributeLocation.PROFILE_ATTRIBUTE, claimName, entry.getValue()); + } + } + } catch (Exception e) { + logger.warn("Cannot parse access token claims", e); + } + } + + return profile; + } + +} diff --git a/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java b/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java index 4787fc92a63d9e..cd68912af405ee 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java @@ -37,6 +37,7 @@ public class OidcConfigs extends SsoConfigs { public static final String OIDC_USE_NONCE = "auth.oidc.useNonce"; public static final String OIDC_CUSTOM_PARAM_RESOURCE = "auth.oidc.customParam.resource"; public static final String OIDC_READ_TIMEOUT = "auth.oidc.readTimeout"; + public static final String OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "auth.oidc.extractJwtAccessTokenClaims"; public static final String OIDC_RESOURCE_CLIENT_ROLE = "auth.oidc.resource.clientRole"; /** @@ -70,6 +71,7 @@ public class OidcConfigs extends SsoConfigs { private Optional useNonce; private Optional customParamResource; private String readTimeout; + private Optional extractJwtAccessTokenClaims; private Optional resourceClientRole; @@ -98,5 +100,6 @@ public OidcConfigs(final com.typesafe.config.Config configs) { customParamResource = getOptional(configs, OIDC_CUSTOM_PARAM_RESOURCE); resourceClientRole = getOptional(configs, OIDC_RESOURCE_CLIENT_ROLE); readTimeout = getOptional(configs, OIDC_READ_TIMEOUT, DEFAULT_OIDC_READ_TIMEOUT); + extractJwtAccessTokenClaims = getOptional(configs, OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).map(Boolean::parseBoolean); } } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java index b777a2fd6a7a3d..540451dd80911a 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java @@ -9,6 +9,7 @@ import org.pac4j.oidc.config.OidcConfiguration; import org.pac4j.oidc.credentials.OidcCredentials; import org.pac4j.oidc.profile.OidcProfile; +import org.pac4j.oidc.profile.OidcProfileDefinition; /** @@ -70,6 +71,7 @@ private Client createPac4jClient() { oidcClient.setName(OIDC_CLIENT_NAME); oidcClient.setCallbackUrl(_oidcConfigs.getAuthBaseUrl() + _oidcConfigs.getAuthBaseCallbackPath()); oidcClient.setCallbackUrlResolver(new PathParameterCallbackUrlResolver()); + oidcClient.addAuthorizationGenerator(new OidcAuthorizationGenerator(new OidcProfileDefinition(), _oidcConfigs)); return oidcClient; } } diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java index e428e59c257946..5de463875db51c 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java @@ -9,7 +9,6 @@ import org.pac4j.oidc.profile.creator.OidcProfileCreator; import org.pac4j.oidc.redirect.OidcRedirectActionBuilder; - public class CustomOidcClient extends OidcClient { public CustomOidcClient(final OidcConfiguration configuration) { diff --git a/datahub-frontend/app/client/AuthServiceClient.java b/datahub-frontend/app/client/AuthServiceClient.java index ba4303129e3b23..e32064f6084f6b 100644 --- a/datahub-frontend/app/client/AuthServiceClient.java +++ b/datahub-frontend/app/client/AuthServiceClient.java @@ -23,19 +23,29 @@ public class AuthServiceClient { private static final String GENERATE_SESSION_TOKEN_ENDPOINT = "auth/generateSessionTokenForUser"; + private static final String SIGN_UP_ENDPOINT = "auth/signUp"; + private static final String RESET_NATIVE_USER_CREDENTIALS_ENDPOINT = "auth/resetNativeUserCredentials"; + private static final String VERIFY_NATIVE_USER_CREDENTIALS_ENDPOINT = "auth/verifyNativeUserCredentials"; private static final String ACCESS_TOKEN_FIELD = "accessToken"; private static final String USER_ID_FIELD = "userId"; + private static final String USER_URN_FIELD = "userUrn"; + private static final String FULL_NAME_FIELD = "fullName"; + private static final String EMAIL_FIELD = "email"; + private static final String TITLE_FIELD = "title"; + private static final String PASSWORD_FIELD = "password"; + private static final String INVITE_TOKEN_FIELD = "inviteToken"; + private static final String RESET_TOKEN_FIELD = "resetToken"; + private static final String IS_NATIVE_USER_CREATED_FIELD = "isNativeUserCreated"; + private static final String ARE_NATIVE_USER_CREDENTIALS_RESET_FIELD = "areNativeUserCredentialsReset"; + private static final String DOES_PASSWORD_MATCH_FIELD = "doesPasswordMatch"; private final String metadataServiceHost; private final Integer metadataServicePort; private final Boolean metadataServiceUseSsl; private final Authentication systemAuthentication; - public AuthServiceClient( - @Nonnull final String metadataServiceHost, - @Nonnull final Integer metadataServicePort, - @Nonnull final Boolean useSsl, - @Nonnull final Authentication systemAuthentication) { + public AuthServiceClient(@Nonnull final String metadataServiceHost, @Nonnull final Integer metadataServicePort, + @Nonnull final Boolean useSsl, @Nonnull final Authentication systemAuthentication) { this.metadataServiceHost = Objects.requireNonNull(metadataServiceHost); this.metadataServicePort = Objects.requireNonNull(metadataServicePort); this.metadataServiceUseSsl = Objects.requireNonNull(useSsl); @@ -88,6 +98,154 @@ public String generateSessionTokenForUser(@Nonnull final String userId) { } } + /** + * Call the Auth Service to create a native Datahub user. + */ + @Nonnull + public boolean signUp(@Nonnull final String userUrn, @Nonnull final String fullName, @Nonnull final String email, + @Nonnull final String title, @Nonnull final String password, @Nonnull final String inviteToken) { + Objects.requireNonNull(userUrn, "userUrn must not be null"); + Objects.requireNonNull(fullName, "fullName must not be null"); + Objects.requireNonNull(email, "email must not be null"); + Objects.requireNonNull(title, "title must not be null"); + Objects.requireNonNull(password, "password must not be null"); + Objects.requireNonNull(inviteToken, "inviteToken must not be null"); + CloseableHttpClient httpClient = HttpClients.createDefault(); + + try { + + final String protocol = this.metadataServiceUseSsl ? "https" : "http"; + final HttpPost request = + new HttpPost(String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort, + SIGN_UP_ENDPOINT)); + + // Build JSON request to verify credentials for a native user. + String json = + String.format("{ \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\" }", + USER_URN_FIELD, userUrn, FULL_NAME_FIELD, fullName, EMAIL_FIELD, email, TITLE_FIELD, title, + PASSWORD_FIELD, password, INVITE_TOKEN_FIELD, inviteToken); + request.setEntity(new StringEntity(json)); + + // Add authorization header with DataHub frontend system id and secret. + request.addHeader(Http.HeaderNames.AUTHORIZATION, this.systemAuthentication.getCredentials()); + + CloseableHttpResponse response = httpClient.execute(request); + final HttpEntity entity = response.getEntity(); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) { + // Successfully generated a token for the User + final String jsonStr = EntityUtils.toString(entity); + return getIsNativeUserCreatedFromJson(jsonStr); + } else { + throw new RuntimeException( + String.format("Bad response from the Metadata Service: %s %s", response.getStatusLine().toString(), + response.getEntity().toString())); + } + } catch (Exception e) { + throw new RuntimeException("Failed to create user", e); + } finally { + try { + httpClient.close(); + } catch (Exception e) { + log.warn("Failed to close http client", e); + } + } + } + + /** + * Call the Auth Service to reset credentials for a native DataHub user. + */ + @Nonnull + public boolean resetNativeUserCredentials(@Nonnull final String userUrn, @Nonnull final String password, + @Nonnull final String resetToken) { + Objects.requireNonNull(userUrn, "userUrn must not be null"); + Objects.requireNonNull(password, "password must not be null"); + Objects.requireNonNull(resetToken, "reset token must not be null"); + CloseableHttpClient httpClient = HttpClients.createDefault(); + + try { + + final String protocol = this.metadataServiceUseSsl ? "https" : "http"; + final HttpPost request = new HttpPost( + String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort, + RESET_NATIVE_USER_CREDENTIALS_ENDPOINT)); + + // Build JSON request to verify credentials for a native user. + String json = + String.format("{ \"%s\":\"%s\", \"%s\":\"%s\", \"%s\":\"%s\" }", USER_URN_FIELD, userUrn, + PASSWORD_FIELD, password, RESET_TOKEN_FIELD, resetToken); + request.setEntity(new StringEntity(json)); + + // Add authorization header with DataHub frontend system id and secret. + request.addHeader(Http.HeaderNames.AUTHORIZATION, this.systemAuthentication.getCredentials()); + + CloseableHttpResponse response = httpClient.execute(request); + final HttpEntity entity = response.getEntity(); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) { + // Successfully generated a token for the User + final String jsonStr = EntityUtils.toString(entity); + return getAreNativeUserCredentialsResetFromJson(jsonStr); + } else { + throw new RuntimeException( + String.format("Bad response from the Metadata Service: %s %s", response.getStatusLine().toString(), + response.getEntity().toString())); + } + } catch (Exception e) { + throw new RuntimeException("Failed to reset credentials for user", e); + } finally { + try { + httpClient.close(); + } catch (Exception e) { + log.warn("Failed to close http client", e); + } + } + } + + /** + * Call the Auth Service to verify the credentials for a native Datahub user. + */ + @Nonnull + public boolean verifyNativeUserCredentials(@Nonnull final String userUrn, @Nonnull final String password) { + Objects.requireNonNull(userUrn, "userUrn must not be null"); + Objects.requireNonNull(password, "password must not be null"); + CloseableHttpClient httpClient = HttpClients.createDefault(); + + try { + + final String protocol = this.metadataServiceUseSsl ? "https" : "http"; + final HttpPost request = new HttpPost( + String.format("%s://%s:%s/%s", protocol, this.metadataServiceHost, this.metadataServicePort, + VERIFY_NATIVE_USER_CREDENTIALS_ENDPOINT)); + + // Build JSON request to verify credentials for a native user. + String json = + String.format("{ \"%s\":\"%s\", \"%s\":\"%s\" }", USER_URN_FIELD, userUrn, PASSWORD_FIELD, password); + request.setEntity(new StringEntity(json)); + + // Add authorization header with DataHub frontend system id and secret. + request.addHeader(Http.HeaderNames.AUTHORIZATION, this.systemAuthentication.getCredentials()); + + CloseableHttpResponse response = httpClient.execute(request); + final HttpEntity entity = response.getEntity(); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) { + // Successfully generated a token for the User + final String jsonStr = EntityUtils.toString(entity); + return getDoesPasswordMatchFromJson(jsonStr); + } else { + throw new RuntimeException( + String.format("Bad response from the Metadata Service: %s %s", response.getStatusLine().toString(), + response.getEntity().toString())); + } + } catch (Exception e) { + throw new RuntimeException("Failed to verify credentials for user", e); + } finally { + try { + httpClient.close(); + } catch (Exception e) { + log.warn("Failed to close http client", e); + } + } + } + private String getAccessTokenFromJson(final String jsonStr) { ObjectMapper mapper = new ObjectMapper(); try { @@ -97,4 +255,31 @@ private String getAccessTokenFromJson(final String jsonStr) { throw new IllegalArgumentException("Failed to parse JSON received from the MetadataService!"); } } + + private boolean getIsNativeUserCreatedFromJson(final String jsonStr) { + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readTree(jsonStr).get(IS_NATIVE_USER_CREATED_FIELD).asBoolean(); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse JSON received from the MetadataService!"); + } + } + + private boolean getAreNativeUserCredentialsResetFromJson(final String jsonStr) { + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readTree(jsonStr).get(ARE_NATIVE_USER_CREDENTIALS_RESET_FIELD).asBoolean(); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse JSON received from the MetadataService!"); + } + } + + private boolean getDoesPasswordMatchFromJson(final String jsonStr) { + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readTree(jsonStr).get(DOES_PASSWORD_MATCH_FIELD).asBoolean(); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse JSON received from the MetadataService!"); + } + } } diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java index 048c8e6ac3cab8..c43d806f593524 100644 --- a/datahub-frontend/app/controllers/AuthenticationController.java +++ b/datahub-frontend/app/controllers/AuthenticationController.java @@ -23,12 +23,12 @@ import play.mvc.Result; import auth.AuthUtils; import auth.JAASConfigs; +import auth.NativeAuthenticationConfigs; import auth.sso.SsoManager; import security.AuthenticationManager; import javax.annotation.Nonnull; import javax.inject.Inject; -import javax.naming.NamingException; import java.time.Duration; import java.time.temporal.ChronoUnit; @@ -45,6 +45,7 @@ public class AuthenticationController extends Controller { private final Logger _logger = LoggerFactory.getLogger(AuthenticationController.class.getName()); private final Config _configs; private final JAASConfigs _jaasConfigs; + private final NativeAuthenticationConfigs _nativeAuthenticationConfigs; @Inject private org.pac4j.core.config.Config _ssoConfig; @@ -62,6 +63,7 @@ public class AuthenticationController extends Controller { public AuthenticationController(@Nonnull Config configs) { _configs = configs; _jaasConfigs = new JAASConfigs(configs); + _nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs); } /** @@ -87,9 +89,10 @@ public Result authenticate() { return redirectToIdentityProvider(); } - // 2. If JAAS auth is enabled, fallback to it - if (_jaasConfigs.isJAASEnabled()) { - return redirect(LOGIN_ROUTE + String.format("?%s=%s", AUTH_REDIRECT_URI_PARAM, encodeRedirectUri(redirectPath))); + // 2. If either JAAS auth or Native auth is enabled, fallback to it + if (_jaasConfigs.isJAASEnabled() || _nativeAuthenticationConfigs.isNativeAuthenticationEnabled()) { + return redirect( + LOGIN_ROUTE + String.format("?%s=%s", AUTH_REDIRECT_URI_PARAM, encodeRedirectUri(redirectPath))); } // 3. If no auth enabled, fallback to using default user account & redirect. @@ -109,9 +112,15 @@ public Result authenticate() { */ @Nonnull public Result logIn() { - if (!_jaasConfigs.isJAASEnabled()) { + boolean jaasEnabled = _jaasConfigs.isJAASEnabled(); + _logger.debug(String.format("Jaas authentication enabled: %b", jaasEnabled)); + boolean nativeAuthenticationEnabled = _nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); + _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); + boolean noAuthEnabled = !jaasEnabled && !nativeAuthenticationEnabled; + if (noAuthEnabled) { + String message = "Neither JAAS nor native authentication is enabled on the server."; final ObjectNode error = Json.newObject(); - error.put("message", "JAAS authentication is not enabled on the server."); + error.put("message", message); return badRequest(error); } @@ -120,23 +129,19 @@ public Result logIn() { final String password = json.findPath(PASSWORD).textValue(); if (StringUtils.isBlank(username)) { - JsonNode invalidCredsJson = Json.newObject() - .put("message", "User name must not be empty."); + JsonNode invalidCredsJson = Json.newObject().put("message", "User name must not be empty."); return badRequest(invalidCredsJson); } ctx().session().clear(); - try { - AuthenticationManager.authenticateUser(username, password); - } catch (NamingException e) { - _logger.error("Authentication error", e); - JsonNode invalidCredsJson = Json.newObject() - .put("message", "Invalid Credentials"); + JsonNode invalidCredsJson = Json.newObject().put("message", "Invalid Credentials"); + boolean loginSucceeded = tryLogin(username, password); + + if (!loginSucceeded) { return badRequest(invalidCredsJson); } - final Urn actorUrn = new CorpuserUrn(username); final String accessToken = _authClient.generateSessionTokenForUser(actorUrn.getId()); ctx().session().put(ACTOR, actorUrn.toString()); @@ -147,6 +152,119 @@ public Result logIn() { .build()); } + /** + * Sign up a native user based on a name, email, title, and password. The invite token must match the global invite + * token stored for the DataHub instance. + * + */ + @Nonnull + public Result signUp() { + boolean nativeAuthenticationEnabled = _nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); + _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); + if (!nativeAuthenticationEnabled) { + String message = "Native authentication is not enabled on the server."; + final ObjectNode error = Json.newObject(); + error.put("message", message); + return badRequest(error); + } + + final JsonNode json = request().body().asJson(); + final String fullName = json.findPath(FULL_NAME).textValue(); + final String email = json.findPath(EMAIL).textValue(); + final String title = json.findPath(TITLE).textValue(); + final String password = json.findPath(PASSWORD).textValue(); + final String inviteToken = json.findPath(INVITE_TOKEN).textValue(); + + if (StringUtils.isBlank(fullName)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Full name must not be empty."); + return badRequest(invalidCredsJson); + } + + if (StringUtils.isBlank(email)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty."); + return badRequest(invalidCredsJson); + } + + if (StringUtils.isBlank(password)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Password must not be empty."); + return badRequest(invalidCredsJson); + } + + if (StringUtils.isBlank(title)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Title must not be empty."); + return badRequest(invalidCredsJson); + } + + if (StringUtils.isBlank(inviteToken)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Invite token must not be empty."); + return badRequest(invalidCredsJson); + } + + ctx().session().clear(); + + final Urn userUrn = new CorpuserUrn(email); + final String userUrnString = userUrn.toString(); + boolean isNativeUserCreated = _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken); + final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId()); + ctx().session().put(ACTOR, userUrnString); + ctx().session().put(ACCESS_TOKEN, accessToken); + return ok().withCookies(Http.Cookie.builder(ACTOR, userUrnString) + .withHttpOnly(false) + .withMaxAge(Duration.of(30, ChronoUnit.DAYS)) + .build()); + } + + /** + * Create a native user based on a name, email, and password. + * + */ + @Nonnull + public Result resetNativeUserCredentials() { + boolean nativeAuthenticationEnabled = _nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); + _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); + if (!nativeAuthenticationEnabled) { + String message = "Native authentication is not enabled on the server."; + final ObjectNode error = Json.newObject(); + error.put("message", message); + return badRequest(error); + } + + final JsonNode json = request().body().asJson(); + final String email = json.findPath(EMAIL).textValue(); + final String password = json.findPath(PASSWORD).textValue(); + final String resetToken = json.findPath(RESET_TOKEN).textValue(); + + if (StringUtils.isBlank(email)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty."); + return badRequest(invalidCredsJson); + } + + if (StringUtils.isBlank(password)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Password must not be empty."); + return badRequest(invalidCredsJson); + } + + if (StringUtils.isBlank(resetToken)) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Reset token must not be empty."); + return badRequest(invalidCredsJson); + } + + ctx().session().clear(); + + final Urn userUrn = new CorpuserUrn(email); + final String userUrnString = userUrn.toString(); + boolean areNativeUserCredentialsReset = + _authClient.resetNativeUserCredentials(userUrnString, password, resetToken); + _logger.debug(String.format("Are native user credentials reset: %b", areNativeUserCredentialsReset)); + final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId()); + ctx().session().put(ACTOR, userUrnString); + ctx().session().put(ACCESS_TOKEN, accessToken); + return ok().withCookies(Http.Cookie.builder(ACTOR, userUrnString) + .withHttpOnly(false) + .withMaxAge(Duration.of(30, ChronoUnit.DAYS)) + .build()); + } + private Result redirectToIdentityProvider() { final PlayWebContext playWebContext = new PlayWebContext(ctx(), _playSessionStore); final Client client = _ssoManager.getSsoProvider().client(); @@ -168,4 +286,30 @@ private String encodeRedirectUri(final String redirectUri) { throw new RuntimeException(String.format("Failed to encode redirect URI %s", redirectUri), e); } } + + private boolean tryLogin(String username, String password) { + JsonNode invalidCredsJson = Json.newObject().put("message", "Invalid Credentials"); + boolean loginSucceeded = false; + + // First try jaas login, if enabled + if (_jaasConfigs.isJAASEnabled()) { + try { + _logger.debug("Attempting jaas authentication"); + AuthenticationManager.authenticateJaasUser(username, password); + loginSucceeded = true; + _logger.debug("Jaas authentication successful"); + } catch (Exception e) { + _logger.debug("Jaas authentication error", e); + } + } + + // If jaas login fails or is disabled, try native auth login + if (_nativeAuthenticationConfigs.isNativeAuthenticationEnabled() && !loginSucceeded) { + final Urn userUrn = new CorpuserUrn(username); + final String userUrnString = userUrn.toString(); + loginSucceeded = loginSucceeded || _authClient.verifyNativeUserCredentials(userUrnString, password); + } + + return loginSucceeded; + } } \ No newline at end of file diff --git a/datahub-frontend/app/security/AuthenticationManager.java b/datahub-frontend/app/security/AuthenticationManager.java index 8103d4a8c2d151..3845438acdbd5c 100644 --- a/datahub-frontend/app/security/AuthenticationManager.java +++ b/datahub-frontend/app/security/AuthenticationManager.java @@ -23,7 +23,7 @@ private AuthenticationManager() { } - public static void authenticateUser(@Nonnull String userName, @Nonnull String password) throws NamingException { + public static void authenticateJaasUser(@Nonnull String userName, @Nonnull String password) throws NamingException { Preconditions.checkArgument(!StringUtils.isAnyEmpty(userName), "Username cannot be empty"); try { JAASLoginService jaasLoginService = new JAASLoginService("WHZ-Authentication"); diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index 3a17b694fbba32..fc15fd0540fa1f 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -39,6 +39,11 @@ play.http.server.akka.max-header-count = ${?DATAHUB_AKKA_MAX_HEADER_COUNT} play.server.akka.max-header-value-length = 8k play.server.akka.max-header-value-length = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH} +# pac4j configuration +# default to PlayCookieSessionStore to keep datahub-frontend's statelessness +pac4j.sessionStore.provider= "PlayCookieSessionStore" +pac4j.sessionStore.provider= ${?PAC4J_SESSIONSTORE_PROVIDER} + # Database configuration # ~~~~~ # You can declare as many datasources as you want. @@ -140,6 +145,7 @@ auth.oidc.useNonce = ${?AUTH_OIDC_USE_NONCE} auth.oidc.customParam.resource = ${?AUTH_OIDC_CUSTOM_PARAM_RESOURCE} auth.oidc.resource.clientRole = ${?AUTH_OIDC_RESOURCE_CLIENT_ROLE} auth.oidc.readTimeout = ${?AUTH_OIDC_READ_TIMEOUT} +auth.oidc.extractJwtAccessTokenClaims = ${?AUTH_OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS} # Whether to extract claims from JWT access token. Defaults to false. # # By default, the callback URL that should be registered with the identity provider is computed as {$baseUrl}/callback/oidc. @@ -153,11 +159,13 @@ auth.oidc.readTimeout = ${?AUTH_OIDC_READ_TIMEOUT} # any username / password combination as valid credentials. To disable this entry point altogether, specify the following config: # auth.jaas.enabled = ${?AUTH_JAAS_ENABLED} +auth.native.enabled = ${?AUTH_NATIVE_ENABLED} # -# To disable all authentication to the app, and proxy all users through a master "datahub" account, make sure that both -# jaas and oidc auth are disabled: +# To disable all authentication to the app, and proxy all users through a master "datahub" account, make sure that, +# jaas, native and oidc auth are disabled: # # auth.jaas.enabled = false +# auth.native.enabled = false # auth.oidc.enabled = false # (or simply omit oidc configurations) # Login session expiration time diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 968c350c379eca..78ddc9c29c440b 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -14,6 +14,8 @@ GET /config co # Authentication in React GET /authenticate controllers.AuthenticationController.authenticate() POST /logIn controllers.AuthenticationController.logIn() +POST /signUp controllers.AuthenticationController.signUp() +POST /resetNativeUserCredentials controllers.AuthenticationController.resetNativeUserCredentials() GET /callback/:protocol controllers.SsoCallbackController.handleCallback(protocol: String) POST /callback/:protocol controllers.SsoCallbackController.handleCallback(protocol: String) GET /logOut controllers.CentralLogoutController.executeLogout() diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index 691722bb568fb6..b87ca1eaeb22e8 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -41,16 +41,20 @@ dependencies { implementation(externalDependency.pac4j) { exclude group: "net.minidev", module: "json-smart" + exclude group: "com.nimbusds", module: "nimbus-jose-jwt" } + implementation "com.nimbusds:nimbus-jose-jwt:7.9" implementation externalDependency.jsonSmart implementation externalDependency.playPac4j implementation externalDependency.shiroCore implementation externalDependency.playEhcache implementation externalDependency.playCache + implementation externalDependency.playEhcache implementation externalDependency.playWs implementation externalDependency.playServer implementation externalDependency.playAkkaHttpServer implementation externalDependency.kafkaClients + implementation externalDependency.awsMskIamAuth testImplementation externalDependency.mockito testImplementation externalDependency.playTest diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 07439350f6323d..d9bc2fe1a864a7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -1,10 +1,13 @@ package com.linkedin.datahub.graphql; import com.datahub.authentication.AuthenticationConfiguration; +import com.datahub.authentication.group.GroupService; import com.datahub.authentication.token.StatefulTokenService; +import com.datahub.authentication.user.NativeUserService; import com.datahub.authorization.AuthorizationConfiguration; import com.google.common.collect.ImmutableList; import com.linkedin.common.VersionedUrn; +import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.analytics.resolver.AnalyticsChartTypeResolver; import com.linkedin.datahub.graphql.analytics.resolver.GetChartsResolver; @@ -28,11 +31,14 @@ import com.linkedin.datahub.graphql.generated.CorpUserInfo; import com.linkedin.datahub.graphql.generated.Dashboard; import com.linkedin.datahub.graphql.generated.DashboardInfo; +import com.linkedin.datahub.graphql.generated.DashboardStatsSummary; +import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts; import com.linkedin.datahub.graphql.generated.DataFlow; import com.linkedin.datahub.graphql.generated.DataJob; import com.linkedin.datahub.graphql.generated.DataJobInputOutput; import com.linkedin.datahub.graphql.generated.DataPlatformInstance; import com.linkedin.datahub.graphql.generated.Dataset; +import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityRelationship; import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy; @@ -62,6 +68,7 @@ import com.linkedin.datahub.graphql.generated.RecommendationContent; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; import com.linkedin.datahub.graphql.generated.SearchResult; +import com.linkedin.datahub.graphql.generated.SiblingProperties; import com.linkedin.datahub.graphql.generated.Test; import com.linkedin.datahub.graphql.generated.TestResult; import com.linkedin.datahub.graphql.generated.UserUsageCounts; @@ -75,23 +82,31 @@ import com.linkedin.datahub.graphql.resolvers.auth.RevokeAccessTokenResolver; import com.linkedin.datahub.graphql.resolvers.browse.BrowsePathsResolver; import com.linkedin.datahub.graphql.resolvers.browse.BrowseResolver; +import com.linkedin.datahub.graphql.resolvers.chart.ChartStatsSummaryResolver; import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver; import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver; import com.linkedin.datahub.graphql.resolvers.container.ParentContainersResolver; +import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardStatsSummaryResolver; +import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsResolver; import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver; +import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver; +import com.linkedin.datahub.graphql.resolvers.dataset.DatasetUsageStatsResolver; import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver; import com.linkedin.datahub.graphql.resolvers.domain.CreateDomainResolver; +import com.linkedin.datahub.graphql.resolvers.domain.DeleteDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.DomainEntitiesResolver; import com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver; import com.linkedin.datahub.graphql.resolvers.domain.SetDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.UnsetDomainResolver; import com.linkedin.datahub.graphql.resolvers.entity.EntityExistsResolver; +import com.linkedin.datahub.graphql.resolvers.glossary.AddRelatedTermsResolver; import com.linkedin.datahub.graphql.resolvers.glossary.CreateGlossaryNodeResolver; import com.linkedin.datahub.graphql.resolvers.glossary.CreateGlossaryTermResolver; import com.linkedin.datahub.graphql.resolvers.glossary.DeleteGlossaryEntityResolver; import com.linkedin.datahub.graphql.resolvers.glossary.GetRootGlossaryNodesResolver; import com.linkedin.datahub.graphql.resolvers.glossary.GetRootGlossaryTermsResolver; import com.linkedin.datahub.graphql.resolvers.glossary.ParentNodesResolver; +import com.linkedin.datahub.graphql.resolvers.glossary.RemoveRelatedTermsResolver; import com.linkedin.datahub.graphql.resolvers.group.AddGroupMembersResolver; import com.linkedin.datahub.graphql.resolvers.group.CreateGroupResolver; import com.linkedin.datahub.graphql.resolvers.group.EntityCountsResolver; @@ -100,6 +115,7 @@ import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.CancelIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionExecutionRequestResolver; +import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateTestConnectionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.GetIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.IngestionSourceExecutionRequestsResolver; import com.linkedin.datahub.graphql.resolvers.ingest.secret.CreateSecretResolver; @@ -121,7 +137,6 @@ import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeResolver; import com.linkedin.datahub.graphql.resolvers.load.OwnerTypeResolver; import com.linkedin.datahub.graphql.resolvers.load.TimeSeriesAspectResolver; -import com.linkedin.datahub.graphql.resolvers.load.UsageTypeResolver; import com.linkedin.datahub.graphql.resolvers.mutate.AddLinkResolver; import com.linkedin.datahub.graphql.resolvers.mutate.AddOwnerResolver; import com.linkedin.datahub.graphql.resolvers.mutate.AddOwnersResolver; @@ -129,15 +144,25 @@ import com.linkedin.datahub.graphql.resolvers.mutate.AddTagsResolver; import com.linkedin.datahub.graphql.resolvers.mutate.AddTermResolver; import com.linkedin.datahub.graphql.resolvers.mutate.AddTermsResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeBatchResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddOwnersResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddTagsResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddTermsResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveOwnersResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveTagsResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveTermsResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateDeprecationResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchSetDomainResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateSoftDeletedResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeResolver; import com.linkedin.datahub.graphql.resolvers.mutate.RemoveLinkResolver; import com.linkedin.datahub.graphql.resolvers.mutate.RemoveOwnerResolver; import com.linkedin.datahub.graphql.resolvers.mutate.RemoveTagResolver; import com.linkedin.datahub.graphql.resolvers.mutate.RemoveTermResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateDescriptionResolver; -import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver; +import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver; import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver; import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver; import com.linkedin.datahub.graphql.resolvers.policy.ListPoliciesResolver; @@ -148,6 +173,8 @@ import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver; import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver; import com.linkedin.datahub.graphql.resolvers.search.SearchResolver; +import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver; +import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver; import com.linkedin.datahub.graphql.resolvers.tag.SetTagColorResolver; import com.linkedin.datahub.graphql.resolvers.test.CreateTestResolver; import com.linkedin.datahub.graphql.resolvers.test.DeleteTestResolver; @@ -155,12 +182,16 @@ import com.linkedin.datahub.graphql.resolvers.test.TestResultsResolver; import com.linkedin.datahub.graphql.resolvers.test.UpdateTestResolver; import com.linkedin.datahub.graphql.resolvers.timeline.GetSchemaBlameResolver; +import com.linkedin.datahub.graphql.resolvers.timeline.GetSchemaVersionListResolver; import com.linkedin.datahub.graphql.resolvers.type.AspectInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.EntityInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver; +import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserInviteTokenResolver; +import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver; +import com.linkedin.datahub.graphql.resolvers.user.GetNativeUserInviteTokenResolver; import com.linkedin.datahub.graphql.resolvers.user.ListUsersResolver; import com.linkedin.datahub.graphql.resolvers.user.RemoveUserResolver; import com.linkedin.datahub.graphql.resolvers.user.UpdateUserStatusResolver; @@ -173,6 +204,7 @@ import com.linkedin.datahub.graphql.types.auth.AccessTokenMetadataType; import com.linkedin.datahub.graphql.types.chart.ChartType; import com.linkedin.datahub.graphql.types.common.mappers.OperationMapper; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.container.ContainerType; import com.linkedin.datahub.graphql.types.corpgroup.CorpGroupType; import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; @@ -196,14 +228,14 @@ import com.linkedin.datahub.graphql.types.notebook.NotebookType; import com.linkedin.datahub.graphql.types.tag.TagType; import com.linkedin.datahub.graphql.types.test.TestType; -import com.linkedin.datahub.graphql.types.usage.UsageType; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.config.DatahubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; -import com.linkedin.metadata.config.VisualConfiguration; import com.linkedin.metadata.config.TestsConfiguration; +import com.linkedin.metadata.config.VisualConfiguration; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.graph.SiblingGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; @@ -217,12 +249,6 @@ import graphql.schema.DataFetchingEnvironment; import graphql.schema.StaticDataFetcher; import graphql.schema.idl.RuntimeWiring; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.dataloader.BatchLoaderContextProvider; -import org.dataloader.DataLoader; -import org.dataloader.DataLoaderOptions; - import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -235,10 +261,15 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.dataloader.BatchLoaderContextProvider; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; import static com.linkedin.datahub.graphql.Constants.*; -import static com.linkedin.metadata.Constants.DATA_PROCESS_INSTANCE_RUN_EVENT_ASPECT_NAME; -import static graphql.Scalars.GraphQLLong; +import static com.linkedin.metadata.Constants.*; +import static graphql.Scalars.*; /** @@ -250,6 +281,7 @@ public class GmsGraphQLEngine { private final EntityClient entityClient; private final GraphClient graphClient; private final UsageClient usageClient; + private final SiblingGraphService siblingGraphService; private final EntityService entityService; private final AnalyticsService analyticsService; @@ -261,6 +293,8 @@ public class GmsGraphQLEngine { private final boolean supportsImpactAnalysis; private final TimeseriesAspectService timeseriesAspectService; private final TimelineService timelineService; + private final NativeUserService nativeUserService; + private final GroupService groupService; private final IngestionConfiguration ingestionConfiguration; private final AuthenticationConfiguration authenticationConfiguration; @@ -287,7 +321,6 @@ public class GmsGraphQLEngine { private final GlossaryTermType glossaryTermType; private final GlossaryNodeType glossaryNodeType; private final AspectType aspectType; - private final UsageType usageType; private final ContainerType containerType; private final DomainType domainType; private final NotebookType notebookType; @@ -333,21 +366,18 @@ public GmsGraphQLEngine( final TimeseriesAspectService timeseriesAspectService, final EntityRegistry entityRegistry, final SecretService secretService, - final IngestionConfiguration ingestionConfiguration, + final NativeUserService nativeUserService, final IngestionConfiguration ingestionConfiguration, final AuthenticationConfiguration authenticationConfiguration, - final AuthorizationConfiguration authorizationConfiguration, - final GitVersion gitVersion, - final TimelineService timelineService, - final boolean supportsImpactAnalysis, - final VisualConfiguration visualConfiguration, - final TelemetryConfiguration telemetryConfiguration, - final TestsConfiguration testsConfiguration, - final DatahubConfiguration datahubConfiguration - ) { + final AuthorizationConfiguration authorizationConfiguration, final GitVersion gitVersion, + final TimelineService timelineService, final boolean supportsImpactAnalysis, + final VisualConfiguration visualConfiguration, final TelemetryConfiguration telemetryConfiguration, + final TestsConfiguration testsConfiguration, final DatahubConfiguration datahubConfiguration, + final SiblingGraphService siblingGraphService, final GroupService groupService) { this.entityClient = entityClient; this.graphClient = graphClient; this.usageClient = usageClient; + this.siblingGraphService = siblingGraphService; this.analyticsService = analyticsService; this.entityService = entityService; @@ -359,6 +389,8 @@ public GmsGraphQLEngine( this.supportsImpactAnalysis = supportsImpactAnalysis; this.timeseriesAspectService = timeseriesAspectService; this.timelineService = timelineService; + this.nativeUserService = nativeUserService; + this.groupService = groupService; this.ingestionConfiguration = Objects.requireNonNull(ingestionConfiguration); this.authenticationConfiguration = Objects.requireNonNull(authenticationConfiguration); @@ -385,7 +417,6 @@ public GmsGraphQLEngine( this.glossaryTermType = new GlossaryTermType(entityClient); this.glossaryNodeType = new GlossaryNodeType(entityClient); this.aspectType = new AspectType(entityClient); - this.usageType = new UsageType(this.usageClient); this.containerType = new ContainerType(entityClient); this.domainType = new DomainType(entityClient); this.notebookType = new NotebookType(entityClient); @@ -492,7 +523,6 @@ public GraphQLEngine.Builder builder() { .addSchema(fileBasedSchema(TESTS_SCHEMA_FILE)) .addDataLoaders(loaderSuppliers(loadableTypes)) .addDataLoader("Aspect", context -> createDataLoader(aspectType, context)) - .addDataLoader("UsageQueryResult", context -> createDataLoader(usageType, context)) .configureRuntimeWiring(this::configureRuntimeWiring); } @@ -525,10 +555,6 @@ private void configureContainerResolvers(final RuntimeWiring.Builder builder) { .type("Container", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("entities", new ContainerEntitiesResolver(entityClient)) - .dataFetcher("domain", new LoadableTypeResolver<>(domainType, (env) -> { - final Container container = env.getSource(); - return container.getDomain() != null ? container.getDomain().getUrn() : null; - })) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((Container) env.getSource()).getPlatform().getUrn())) @@ -618,14 +644,29 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("ingestionSource", new GetIngestionSourceResolver(this.entityClient)) .dataFetcher("executionRequest", new GetIngestionExecutionRequestResolver(this.entityClient)) .dataFetcher("getSchemaBlame", new GetSchemaBlameResolver(this.timelineService)) + .dataFetcher("getSchemaVersionList", new GetSchemaVersionListResolver(this.timelineService)) .dataFetcher("test", getResolver(testType)) .dataFetcher("listTests", new ListTestsResolver(entityClient)) .dataFetcher("getRootGlossaryTerms", new GetRootGlossaryTermsResolver(this.entityClient)) .dataFetcher("getRootGlossaryNodes", new GetRootGlossaryNodesResolver(this.entityClient)) .dataFetcher("entityExists", new EntityExistsResolver(this.entityService)) + .dataFetcher("getNativeUserInviteToken", new GetNativeUserInviteTokenResolver(this.nativeUserService)) + .dataFetcher("entity", getEntityResolver()) ); } + private DataFetcher getEntityResolver() { + return new EntityTypeResolver(entityTypes, + (env) -> { + try { + Urn urn = Urn.createFromString(env.getArgument(URN_FIELD_NAME)); + return UrnToEntityMapper.map(urn); + } catch (Exception e) { + throw new RuntimeException("Failed to get entity", e); + } + }); + } + private DataFetcher getResolver(LoadableType loadableType) { return getResolver(loadableType, this::getUrnField); } @@ -642,8 +683,11 @@ private String getUrnField(DataFetchingEnvironment env) { private void configureMutationResolvers(final RuntimeWiring.Builder builder) { builder.type("Mutation", typeWiring -> typeWiring .dataFetcher("updateDataset", new MutableTypeResolver<>(datasetType)) + .dataFetcher("updateDatasets", new MutableTypeBatchResolver<>(datasetType)) + .dataFetcher("createTag", new CreateTagResolver(this.entityClient)) .dataFetcher("updateTag", new MutableTypeResolver<>(tagType)) .dataFetcher("setTagColor", new SetTagColorResolver(entityClient, entityService)) + .dataFetcher("deleteTag", new DeleteTagResolver(entityClient)) .dataFetcher("updateChart", new MutableTypeResolver<>(chartType)) .dataFetcher("updateDashboard", new MutableTypeResolver<>(dashboardType)) .dataFetcher("updateNotebook", new MutableTypeResolver<>(notebookType)) @@ -653,28 +697,37 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType)) .dataFetcher("addTag", new AddTagResolver(entityService)) .dataFetcher("addTags", new AddTagsResolver(entityService)) + .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService)) .dataFetcher("removeTag", new RemoveTagResolver(entityService)) + .dataFetcher("batchRemoveTags", new BatchRemoveTagsResolver(entityService)) .dataFetcher("addTerm", new AddTermResolver(entityService)) + .dataFetcher("batchAddTerms", new BatchAddTermsResolver(entityService)) .dataFetcher("addTerms", new AddTermsResolver(entityService)) .dataFetcher("removeTerm", new RemoveTermResolver(entityService)) + .dataFetcher("batchRemoveTerms", new BatchRemoveTermsResolver(entityService)) .dataFetcher("createPolicy", new UpsertPolicyResolver(this.entityClient)) .dataFetcher("updatePolicy", new UpsertPolicyResolver(this.entityClient)) .dataFetcher("deletePolicy", new DeletePolicyResolver(this.entityClient)) .dataFetcher("updateDescription", new UpdateDescriptionResolver(entityService)) .dataFetcher("addOwner", new AddOwnerResolver(entityService)) .dataFetcher("addOwners", new AddOwnersResolver(entityService)) + .dataFetcher("batchAddOwners", new BatchAddOwnersResolver(entityService)) .dataFetcher("removeOwner", new RemoveOwnerResolver(entityService)) + .dataFetcher("batchRemoveOwners", new BatchRemoveOwnersResolver(entityService)) .dataFetcher("addLink", new AddLinkResolver(entityService)) .dataFetcher("removeLink", new RemoveLinkResolver(entityService)) - .dataFetcher("addGroupMembers", new AddGroupMembersResolver(this.entityClient)) - .dataFetcher("removeGroupMembers", new RemoveGroupMembersResolver(this.entityClient)) - .dataFetcher("createGroup", new CreateGroupResolver(this.entityClient)) + .dataFetcher("addGroupMembers", new AddGroupMembersResolver(this.groupService)) + .dataFetcher("removeGroupMembers", new RemoveGroupMembersResolver(this.groupService)) + .dataFetcher("createGroup", new CreateGroupResolver(this.groupService)) .dataFetcher("removeUser", new RemoveUserResolver(this.entityClient)) .dataFetcher("removeGroup", new RemoveGroupResolver(this.entityClient)) .dataFetcher("updateUserStatus", new UpdateUserStatusResolver(this.entityClient)) .dataFetcher("createDomain", new CreateDomainResolver(this.entityClient)) + .dataFetcher("deleteDomain", new DeleteDomainResolver(entityClient)) .dataFetcher("setDomain", new SetDomainResolver(this.entityClient, this.entityService)) + .dataFetcher("batchSetDomain", new BatchSetDomainResolver(this.entityService)) .dataFetcher("updateDeprecation", new UpdateDeprecationResolver(this.entityClient, this.entityService)) + .dataFetcher("batchUpdateDeprecation", new BatchUpdateDeprecationResolver(entityService)) .dataFetcher("unsetDomain", new UnsetDomainResolver(this.entityClient, this.entityService)) .dataFetcher("createSecret", new CreateSecretResolver(this.entityClient, this.secretService)) .dataFetcher("deleteSecret", new DeleteSecretResolver(this.entityClient)) @@ -685,6 +738,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("deleteIngestionSource", new DeleteIngestionSourceResolver(this.entityClient)) .dataFetcher("createIngestionExecutionRequest", new CreateIngestionExecutionRequestResolver(this.entityClient, this.ingestionConfiguration)) .dataFetcher("cancelIngestionExecutionRequest", new CancelIngestionExecutionRequestResolver(this.entityClient)) + .dataFetcher("createTestConnectionRequest", new CreateTestConnectionRequestResolver(this.entityClient, this.ingestionConfiguration)) .dataFetcher("deleteAssertion", new DeleteAssertionResolver(this.entityClient, this.entityService)) .dataFetcher("createTest", new CreateTestResolver(this.entityClient)) .dataFetcher("updateTest", new UpdateTestResolver(this.entityClient)) @@ -693,8 +747,15 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("createGlossaryTerm", new CreateGlossaryTermResolver(this.entityClient)) .dataFetcher("createGlossaryNode", new CreateGlossaryNodeResolver(this.entityClient)) .dataFetcher("updateParentNode", new UpdateParentNodeResolver(entityService)) - .dataFetcher("deleteGlossaryEntity", new DeleteGlossaryEntityResolver(this.entityClient, this.entityService)) + .dataFetcher("deleteGlossaryEntity", + new DeleteGlossaryEntityResolver(this.entityClient, this.entityService)) .dataFetcher("updateName", new UpdateNameResolver(entityService)) + .dataFetcher("addRelatedTerms", new AddRelatedTermsResolver(this.entityService)) + .dataFetcher("removeRelatedTerms", new RemoveRelatedTermsResolver(this.entityService)) + .dataFetcher("createNativeUserInviteToken", new CreateNativeUserInviteTokenResolver(this.nativeUserService)) + .dataFetcher("createNativeUserResetToken", new CreateNativeUserResetTokenResolver(this.nativeUserService)) + .dataFetcher("batchUpdateSoftDeleted", new BatchUpdateSoftDeletedResolver(this.entityService)) + ); } @@ -778,14 +839,7 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { builder .type("Dataset", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final Dataset dataset = env.getSource(); - return dataset.getDomain() != null ? dataset.getDomain().getUrn() : null; - })) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((Dataset) env.getSource()).getPlatform().getUrn()) ) @@ -817,7 +871,8 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { OperationMapper::map ) ) - .dataFetcher("usageStats", new UsageTypeResolver()) + .dataFetcher("usageStats", new DatasetUsageStatsResolver(this.usageClient)) + .dataFetcher("statsSummary", new DatasetStatsSummaryResolver(this.usageClient)) .dataFetcher("health", new DatasetHealthResolver(graphClient, timeseriesAspectService)) .dataFetcher("schemaMetadata", new AspectResolver()) .dataFetcher("assertions", new EntityAssertionsResolver(entityClient, graphClient)) @@ -841,11 +896,27 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("foreignDataset", new LoadableTypeResolver<>(datasetType, (env) -> ((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn())) ) + .type("SiblingProperties", typeWiring -> typeWiring + .dataFetcher("siblings", + new EntityTypeBatchResolver( + new ArrayList<>(entityTypes), + (env) -> ((SiblingProperties) env.getSource()).getSiblings())) + ) .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring .dataFetcher("author", new LoadableTypeResolver<>(corpUserType, (env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())) + ) + .type("DatasetStatsSummary", typeWiring -> typeWiring + .dataFetcher("topUsersLast30Days", new LoadableTypeBatchResolver<>(corpUserType, + (env) -> { + DatasetStatsSummary summary = ((DatasetStatsSummary) env.getSource()); + return summary.getTopUsersLast30Days() != null + ? summary.getTopUsersLast30Days().stream() + .map(CorpUser::getUrn) + .collect(Collectors.toList()) + : null; + })) ); - } /** @@ -954,9 +1025,7 @@ private void configureNotebookResolvers(final RuntimeWiring.Builder builder) { return notebook.getDataPlatformInstance() != null ? notebook.getDataPlatformInstance().getUrn() : null; }) ) - .dataFetcher("domain", new LoadableTypeResolver<>(domainType, - (env) -> ((Notebook) env.getSource()).getDomain().getUrn()) - )); + ); } /** @@ -965,17 +1034,9 @@ private void configureNotebookResolvers(final RuntimeWiring.Builder builder) { private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { builder.type("Dashboard", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((Dashboard) env.getSource()).getPlatform().getUrn())) - .dataFetcher("domain", new LoadableTypeResolver<>( - domainType, - (env) -> { - final Dashboard dashboard = env.getSource(); - return dashboard.getDomain() != null ? dashboard.getDomain().getUrn() : null; - } - ) - ) .dataFetcher("dataPlatformInstance", new LoadableTypeResolver<>(dataPlatformInstanceType, (env) -> { @@ -990,6 +1051,8 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { }) ) .dataFetcher("parentContainers", new ParentContainersResolver(entityClient)) + .dataFetcher("usageStats", new DashboardUsageStatsResolver(timeseriesAspectService)) + .dataFetcher("statsSummary", new DashboardStatsSummaryResolver(timeseriesAspectService)) ); builder.type("DashboardInfo", typeWiring -> typeWiring .dataFetcher("charts", new LoadableTypeBatchResolver<>(chartType, @@ -997,6 +1060,22 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { .map(Chart::getUrn) .collect(Collectors.toList()))) ); + builder.type("DashboardUserUsageCounts", typeWiring -> typeWiring + .dataFetcher("user", new LoadableTypeResolver<>( + corpUserType, + (env) -> ((DashboardUserUsageCounts) env.getSource()).getUser().getUrn())) + ); + builder.type("DashboardStatsSummary", typeWiring -> typeWiring + .dataFetcher("topUsersLast30Days", new LoadableTypeBatchResolver<>(corpUserType, + (env) -> { + DashboardStatsSummary summary = ((DashboardStatsSummary) env.getSource()); + return summary.getTopUsersLast30Days() != null + ? summary.getTopUsersLast30Days().stream() + .map(CorpUser::getUrn) + .collect(Collectors.toList()) + : null; + })) + ); } /** @@ -1005,16 +1084,9 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { private void configureChartResolvers(final RuntimeWiring.Builder builder) { builder.type("Chart", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((Chart) env.getSource()).getPlatform().getUrn())) - .dataFetcher("domain", new LoadableTypeResolver<>( - domainType, - (env) -> { - final Chart chart = env.getSource(); - return chart.getDomain() != null ? chart.getDomain().getUrn() : null; - }) - ) .dataFetcher("dataPlatformInstance", new LoadableTypeResolver<>(dataPlatformInstanceType, (env) -> { @@ -1030,6 +1102,7 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) { }) ) .dataFetcher("parentContainers", new ParentContainersResolver(entityClient)) + .dataFetcher("statsSummary", new ChartStatsSummaryResolver(this.timeseriesAspectService)) ); builder.type("ChartInfo", typeWiring -> typeWiring .dataFetcher("inputs", new LoadableTypeBatchResolver<>(datasetType, @@ -1089,16 +1162,9 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) { builder .type("DataJob", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("dataFlow", new LoadableTypeResolver<>(dataFlowType, (env) -> ((DataJob) env.getSource()).getDataFlow().getUrn())) - .dataFetcher("domain", new LoadableTypeResolver<>( - domainType, - (env) -> { - final DataJob dataJob = env.getSource(); - return dataJob.getDomain() != null ? dataJob.getDomain().getUrn() : null; - }) - ) .dataFetcher("dataPlatformInstance", new LoadableTypeResolver<>(dataPlatformInstanceType, (env) -> { @@ -1131,16 +1197,9 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) { builder .type("DataFlow", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((DataFlow) env.getSource()).getPlatform().getUrn())) - .dataFetcher("domain", new LoadableTypeResolver<>( - domainType, - (env) -> { - final DataFlow dataFlow = env.getSource(); - return dataFlow.getDomain() != null ? dataFlow.getDomain().getUrn() : null; - }) - ) .dataFetcher("dataPlatformInstance", new LoadableTypeResolver<>(dataPlatformInstanceType, (env) -> { @@ -1158,7 +1217,7 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde builder .type("MLFeatureTable", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((MLFeatureTable) env.getSource()).getPlatform().getUrn())) @@ -1169,12 +1228,6 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde return entity.getDataPlatformInstance() != null ? entity.getDataPlatformInstance().getUrn() : null; }) ) - .dataFetcher("domain", new LoadableTypeResolver<>( - domainType, - (env) -> { - final MLFeatureTable entity = env.getSource(); - return entity.getDomain() != null ? entity.getDomain().getUrn() : null; - })) ) .type("MLFeatureTableProperties", typeWiring -> typeWiring .dataFetcher("mlFeatures", @@ -1208,7 +1261,7 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde ) .type("MLModel", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((MLModel) env.getSource()).getPlatform().getUrn())) .dataFetcher("dataPlatformInstance", @@ -1218,13 +1271,6 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde return mlModel.getDataPlatformInstance() != null ? mlModel.getDataPlatformInstance().getUrn() : null; }) ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final MLModel mlModel = env.getSource(); - return mlModel.getDomain() != null ? mlModel.getDomain().getUrn() : null; - })) ) .type("MLModelProperties", typeWiring -> typeWiring .dataFetcher("groups", new LoadableTypeBatchResolver<>(mlModelGroupType, @@ -1241,7 +1287,7 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde ) .type("MLModelGroup", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType, (env) -> ((MLModelGroup) env.getSource()).getPlatform().getUrn()) ) @@ -1252,17 +1298,10 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde return entity.getDataPlatformInstance() != null ? entity.getDataPlatformInstance().getUrn() : null; }) ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final MLModelGroup entity = env.getSource(); - return entity.getDomain() != null ? entity.getDomain().getUrn() : null; - })) ) .type("MLFeature", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("dataPlatformInstance", new LoadableTypeResolver<>(dataPlatformInstanceType, (env) -> { @@ -1270,17 +1309,10 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde return entity.getDataPlatformInstance() != null ? entity.getDataPlatformInstance().getUrn() : null; }) ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final MLFeature entity = env.getSource(); - return entity.getDomain() != null ? entity.getDomain().getUrn() : null; - })) ) .type("MLPrimaryKey", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("dataPlatformInstance", new LoadableTypeResolver<>(dataPlatformInstanceType, (env) -> { @@ -1288,13 +1320,6 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde return entity.getDataPlatformInstance() != null ? entity.getDataPlatformInstance().getUrn() : null; }) ) - .dataFetcher("domain", - new LoadableTypeResolver<>( - domainType, - (env) -> { - final MLPrimaryKey entity = env.getSource(); - return entity.getDomain() != null ? entity.getDomain().getUrn() : null; - })) ); } @@ -1311,6 +1336,11 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient) ) ); + builder.type("DomainAssociation", typeWiring -> typeWiring + .dataFetcher("domain", + new LoadableTypeResolver<>(domainType, + (env) -> ((com.linkedin.datahub.graphql.generated.DomainAssociation) env.getSource()).getDomain().getUrn())) + ); } private void configureAssertionResolvers(final RuntimeWiring.Builder builder) { @@ -1349,7 +1379,7 @@ private void configurePolicyResolvers(final RuntimeWiring.Builder builder) { private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder builder) { builder.type("DataProcessInstance", typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) - .dataFetcher("lineage", new EntityLineageResultResolver(graphClient)) + .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("state", new TimeSeriesAspectResolver( this.entityClient, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java index 47b000b6ea4ef8..74c4c541b972b1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java @@ -4,15 +4,12 @@ import graphql.ExecutionInput; import graphql.ExecutionResult; import graphql.GraphQL; +import graphql.execution.instrumentation.tracing.TracingInstrumentation; import graphql.schema.GraphQLSchema; import graphql.schema.idl.RuntimeWiring; import graphql.schema.idl.SchemaGenerator; import graphql.schema.idl.SchemaParser; import graphql.schema.idl.TypeDefinitionRegistry; -import org.dataloader.DataLoader; -import org.dataloader.DataLoaderRegistry; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -20,8 +17,12 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderRegistry; -import static graphql.schema.idl.RuntimeWiring.newRuntimeWiring; +import static graphql.schema.idl.RuntimeWiring.*; /** * Simple wrapper around a {@link GraphQL} instance providing APIs for building an engine and executing @@ -61,6 +62,7 @@ private GraphQLEngine(@Nonnull final List schemas, */ _graphQL = new GraphQL.Builder(graphQLSchema) .defaultDataFetcherExceptionHandler(new DataHubDataFetcherExceptionHandler()) + .instrumentation(new TracingInstrumentation()) .build(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index 173d96f804682e..a6ce9650d2bba5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -5,14 +5,27 @@ import com.datahub.authorization.Authorizer; import com.datahub.authorization.ResourceSpec; import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.metadata.authorization.PoliciesConfig; +import java.time.Clock; import java.util.Optional; import javax.annotation.Nonnull; +import static com.linkedin.datahub.graphql.resolvers.AuthUtils.*; +import static com.linkedin.metadata.Constants.*; + public class AuthorizationUtils { + private static final Clock CLOCK = Clock.systemUTC(); + + public static AuditStamp createAuditStamp(@Nonnull QueryContext context) { + return new AuditStamp().setTime(CLOCK.millis()).setActor(UrnUtils.getUrn(context.getActorUrn())); + } + public static boolean canManageUsersAndGroups(@Nonnull QueryContext context) { return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_USERS_AND_GROUPS_PRIVILEGE); } @@ -25,6 +38,24 @@ public static boolean canManageTokens(@Nonnull QueryContext context) { return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_ACCESS_TOKENS); } + /** + * Returns true if the current used is able to create Domains. This is true if the user has the 'Manage Domains' or 'Create Domains' platform privilege. + */ + public static boolean canCreateDomains(@Nonnull QueryContext context) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup( + ImmutableList.of( + new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.CREATE_DOMAINS_PRIVILEGE.getType())), + new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType())) + )); + + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + orPrivilegeGroups); + } + public static boolean canManageDomains(@Nonnull QueryContext context) { return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE); } @@ -33,6 +64,45 @@ public static boolean canManageGlossaries(@Nonnull QueryContext context) { return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE); } + /** + * Returns true if the current used is able to create Tags. This is true if the user has the 'Manage Tags' or 'Create Tags' platform privilege. + */ + public static boolean canCreateTags(@Nonnull QueryContext context) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup( + ImmutableList.of( + new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.CREATE_TAGS_PRIVILEGE.getType())), + new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.MANAGE_TAGS_PRIVILEGE.getType())) + )); + + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + orPrivilegeGroups); + } + + public static boolean canManageTags(@Nonnull QueryContext context) { + return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_TAGS_PRIVILEGE); + } + + public static boolean canDeleteEntity(@Nonnull Urn entityUrn, @Nonnull QueryContext context) { + return isAuthorized(context, Optional.of(new ResourceSpec(entityUrn.getEntityType(), entityUrn.toString())), PoliciesConfig.DELETE_ENTITY_PRIVILEGE); + } + + public static boolean canManageUserCredentials(@Nonnull QueryContext context) { + return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE); + } + + public static boolean canEditGroupMembers(@Nonnull String groupUrnStr, @Nonnull QueryContext context) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup( + ImmutableList.of(ALL_PRIVILEGES_GROUP, + new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.EDIT_GROUP_MEMBERS_PRIVILEGE.getType())))); + + return AuthorizationUtils.isAuthorized(context.getAuthorizer(), context.getActorUrn(), CORP_GROUP_ENTITY_NAME, + groupUrnStr, orPrivilegeGroups); + } + public static boolean isAuthorized( @Nonnull QueryContext context, @Nonnull Optional resourceSpec, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java index 68f25fd14df558..35ad96193263a8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java @@ -5,6 +5,7 @@ import com.datahub.authorization.Authorizer; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.generated.AuthenticatedUser; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.PlatformPrivileges; @@ -19,6 +20,7 @@ import java.util.Collections; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; import static com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils.*; import static com.linkedin.metadata.Constants.*; @@ -63,6 +65,10 @@ public CompletableFuture get(DataFetchingEnvironment environm platformPrivileges.setManageTokens(canManageTokens(context)); platformPrivileges.setManageTests(canManageTests(context)); platformPrivileges.setManageGlossaries(canManageGlossaries(context)); + platformPrivileges.setManageUserCredentials(canManageUserCredentials(context)); + platformPrivileges.setCreateDomains(AuthorizationUtils.canCreateDomains(context)); + platformPrivileges.setCreateTags(AuthorizationUtils.canCreateTags(context)); + platformPrivileges.setManageTags(AuthorizationUtils.canManageTags(context)); // Construct and return authenticated user object. final AuthenticatedUser authUser = new AuthenticatedUser(); @@ -131,6 +137,14 @@ private boolean canManageGlossaries(final QueryContext context) { return isAuthorized(context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_GLOSSARIES_PRIVILEGE); } + /** + * Returns true if the authenticated user has privileges to manage user credentials + */ + private boolean canManageUserCredentials(@Nonnull QueryContext context) { + return isAuthorized(context.getAuthorizer(), context.getActorUrn(), + PoliciesConfig.MANAGE_USER_CREDENTIALS_PRIVILEGE); + } + /** * Returns true if the provided actor is authorized for a particular privilege, false otherwise. */ diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index a396a719ce4948..7ecb8548519c13 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -2,6 +2,7 @@ import com.datahub.authentication.Authentication; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.ValidationException; import com.linkedin.datahub.graphql.generated.FacetFilterInput; @@ -27,6 +28,9 @@ public class ResolverUtils { + private static final Set KEYWORD_EXCLUDED_FILTERS = ImmutableSet.of( + "runId" + ); private static final ObjectMapper MAPPER = new ObjectMapper(); private static final Logger _logger = LoggerFactory.getLogger(ResolverUtils.class.getName()); @@ -56,6 +60,14 @@ public static Authentication getAuthentication(DataFetchingEnvironment environme return ((QueryContext) environment.getContext()).getAuthentication(); } + /** + * @apiNote DO NOT use this method if the facet filters do not include `.keyword` suffix to ensure + * that it is matched against a keyword filter in ElasticSearch. + * + * @param facetFilterInputs The list of facet filters inputs + * @param validFacetFields The set of valid fields against which to filter for. + * @return A map of filter definitions to be used in ElasticSearch. + */ @Nonnull public static Map buildFacetFilters(@Nullable List facetFilterInputs, @Nonnull Set validFacetFields) { @@ -81,7 +93,14 @@ public static Filter buildFilter(@Nullable List facetFilterInp return null; } return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(facetFilterInputs.stream() - .map(filter -> new Criterion().setField(filter.getField() + ESUtils.KEYWORD_SUFFIX).setValue(filter.getValue())) + .map(filter -> new Criterion().setField(getFilterField(filter.getField())).setValue(filter.getValue())) .collect(Collectors.toList()))))); } + + private static String getFilterField(final String originalField) { + if (KEYWORD_EXCLUDED_FILTERS.contains(originalField)) { + return originalField; + } + return originalField + ESUtils.KEYWORD_SUFFIX; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java index c0730485a709e1..95bd9efe71cd8f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/DeleteAssertionResolver.java @@ -14,14 +14,17 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; /** * GraphQL Resolver that deletes an Assertion. */ +@Slf4j public class DeleteAssertionResolver implements DataFetcher> { private final EntityClient _entityClient; @@ -46,6 +49,16 @@ public CompletableFuture get(final DataFetchingEnvironment environment) if (isAuthorizedToDeleteAssertion(context, assertionUrn)) { try { _entityClient.deleteEntity(assertionUrn, context.getAuthentication()); + + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync(() -> { + try { + _entityClient.deleteEntityReferences(assertionUrn, context.getAuthentication()); + } catch (RemoteInvocationException e) { + log.error(String.format("Caught exception while attempting to clear all entity references for assertion with urn %s", assertionUrn), e); + } + }); + return true; } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform delete against assertion with urn %s", assertionUrn), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java index 8a14ef64e6fd52..b6bd4a7d89c89d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java @@ -1,29 +1,24 @@ package com.linkedin.datahub.graphql.resolvers.auth; -import com.google.common.collect.ImmutableSet; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.AccessTokenMetadata; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.ListAccessTokenInput; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; -import com.linkedin.datahub.graphql.generated.AccessTokenMetadata; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.search.utils.QueryUtils; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; - import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; @@ -36,8 +31,6 @@ public class ListAccessTokensResolver implements DataFetcher> { private static final String EXPIRES_AT_FIELD_NAME = "expiresAt"; - private static final Set FACET_FIELDS = - ImmutableSet.of("ownerUrn", "actorUrn", "name", "createdAt", "expiredAt", "description"); private final EntityClient _entityClient; @@ -62,7 +55,7 @@ public CompletableFuture get(DataFetchingEnvironment envi new SortCriterion().setField(EXPIRES_AT_FIELD_NAME).setOrder(SortOrder.DESCENDING); final SearchResult searchResult = _entityClient.search(Constants.ACCESS_TOKEN_ENTITY_NAME, "", - QueryUtils.newFilter(buildFacetFilters(filters, FACET_FIELDS)), sortCriterion, start, count, + buildFilter(filters), sortCriterion, start, count, getAuthentication(environment)); final List tokens = searchResult.getEntities().stream().map(entity -> { @@ -101,6 +94,6 @@ public CompletableFuture get(DataFetchingEnvironment envi */ private boolean isListingSelfTokens(final List filters, final QueryContext context) { return AuthorizationUtils.canGeneratePersonalAccessToken(context) && filters.stream() - .anyMatch(filter -> filter.getField().equals("actorUrn") && filter.getValue().equals(context.getActorUrn())); + .anyMatch(filter -> filter.getField().equals("ownerUrn") && filter.getValue().equals(context.getActorUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/ChartStatsSummaryResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/ChartStatsSummaryResolver.java new file mode 100644 index 00000000000000..207da02de6ec2d --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/ChartStatsSummaryResolver.java @@ -0,0 +1,34 @@ +package com.linkedin.datahub.graphql.resolvers.chart; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.ChartStatsSummary; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class ChartStatsSummaryResolver implements DataFetcher> { + + private final TimeseriesAspectService timeseriesAspectService; + private final Cache summaryCache; + + public ChartStatsSummaryResolver(final TimeseriesAspectService timeseriesAspectService) { + this.timeseriesAspectService = timeseriesAspectService; + this.summaryCache = CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterWrite(6, TimeUnit.HOURS) + .build(); + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + // Not yet implemented + return CompletableFuture.completedFuture(null); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryResolver.java new file mode 100644 index 00000000000000..db125384745a10 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryResolver.java @@ -0,0 +1,101 @@ +package com.linkedin.datahub.graphql.resolvers.dashboard; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics; +import com.linkedin.datahub.graphql.generated.DashboardStatsSummary; +import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*; + +@Slf4j +public class DashboardStatsSummaryResolver implements DataFetcher> { + + // The maximum number of top users to show in the summary stats + private static final Integer MAX_TOP_USERS = 5; + + private final TimeseriesAspectService timeseriesAspectService; + private final Cache summaryCache; + + public DashboardStatsSummaryResolver(final TimeseriesAspectService timeseriesAspectService) { + this.timeseriesAspectService = timeseriesAspectService; + this.summaryCache = CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterWrite(6, TimeUnit.HOURS) // TODO: Make caching duration configurable externally. + .build(); + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); + + return CompletableFuture.supplyAsync(() -> { + + if (this.summaryCache.getIfPresent(resourceUrn) != null) { + return this.summaryCache.getIfPresent(resourceUrn); + } + + try { + + final DashboardStatsSummary result = new DashboardStatsSummary(); + + // Obtain total dashboard view count, by viewing the latest reported dashboard metrics. + List dashboardUsageMetrics = + getDashboardUsageMetrics(resourceUrn.toString(), null, null, 1, this.timeseriesAspectService); + if (dashboardUsageMetrics.size() > 0) { + result.setViewCount(getDashboardViewCount(resourceUrn)); + } + + // Obtain unique user statistics, by rolling up unique users over the past month. + List userUsageCounts = getDashboardUsagePerUser(resourceUrn); + result.setUniqueUserCountLast30Days(userUsageCounts.size()); + result.setTopUsersLast30Days( + trimUsers(userUsageCounts.stream().map(DashboardUserUsageCounts::getUser).collect(Collectors.toList()))); + + this.summaryCache.put(resourceUrn, result); + return result; + + } catch (Exception e) { + log.error(String.format("Failed to load dashboard usage summary for resource %s", resourceUrn.toString()), e); + return null; // Do not throw when loading usage summary fails. + } + }); + } + + private int getDashboardViewCount(final Urn resourceUrn) { + List dashboardUsageMetrics = getDashboardUsageMetrics( + resourceUrn.toString(), + null, + null, + 1, + this.timeseriesAspectService); + return dashboardUsageMetrics.get(0).getViewsCount(); + } + + private List getDashboardUsagePerUser(final Urn resourceUrn) { + long now = System.currentTimeMillis(); + long nowMinusOneMonth = timeMinusOneMonth(now); + Filter bucketStatsFilter = createUsageFilter(resourceUrn.toString(), nowMinusOneMonth, now, true); + return getUserUsageCounts(bucketStatsFilter, this.timeseriesAspectService); + } + + private List trimUsers(final List originalUsers) { + if (originalUsers.size() > MAX_TOP_USERS) { + return originalUsers.subList(0, MAX_TOP_USERS); + } + return originalUsers; + } + } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsResolver.java new file mode 100644 index 00000000000000..ef333abedd0f3a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsResolver.java @@ -0,0 +1,97 @@ +package com.linkedin.datahub.graphql.resolvers.dashboard; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.DashboardUsageAggregation; +import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics; +import com.linkedin.datahub.graphql.generated.DashboardUsageQueryResult; +import com.linkedin.datahub.graphql.generated.DashboardUsageQueryResultAggregations; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.types.dashboard.mappers.DashboardUsageMetricMapper; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.EnvelopedAspect; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*; + + +/** + * Resolver used for resolving the usage statistics of a Dashboard. + *

+ * Returns daily as well as absolute usage metrics of Dashboard + */ +@Slf4j +public class DashboardUsageStatsResolver implements DataFetcher> { + private static final String ES_FIELD_EVENT_GRANULARITY = "eventGranularity"; + private final TimeseriesAspectService timeseriesAspectService; + + public DashboardUsageStatsResolver(TimeseriesAspectService timeseriesAspectService) { + this.timeseriesAspectService = timeseriesAspectService; + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final String dashboardUrn = ((Entity) environment.getSource()).getUrn(); + final Long maybeStartTimeMillis = environment.getArgumentOrDefault("startTimeMillis", null); + final Long maybeEndTimeMillis = environment.getArgumentOrDefault("endTimeMillis", null); + // Max number of aspects to return for absolute dashboard usage. + final Integer maybeLimit = environment.getArgumentOrDefault("limit", null); + + return CompletableFuture.supplyAsync(() -> { + DashboardUsageQueryResult usageQueryResult = new DashboardUsageQueryResult(); + + // Time Bucket Stats + Filter bucketStatsFilter = createUsageFilter(dashboardUrn, maybeStartTimeMillis, maybeEndTimeMillis, true); + List dailyUsageBuckets = getBuckets(bucketStatsFilter, dashboardUrn, timeseriesAspectService); + DashboardUsageQueryResultAggregations aggregations = getAggregations(bucketStatsFilter, dailyUsageBuckets, timeseriesAspectService); + + usageQueryResult.setBuckets(dailyUsageBuckets); + usageQueryResult.setAggregations(aggregations); + + // Absolute usage metrics + List dashboardUsageMetrics = + getDashboardUsageMetrics(dashboardUrn, maybeStartTimeMillis, maybeEndTimeMillis, maybeLimit); + usageQueryResult.setMetrics(dashboardUsageMetrics); + return usageQueryResult; + }); + } + + private List getDashboardUsageMetrics(String dashboardUrn, Long maybeStartTimeMillis, + Long maybeEndTimeMillis, Integer maybeLimit) { + List dashboardUsageMetrics; + try { + Filter filter = new Filter(); + final ArrayList criteria = new ArrayList<>(); + + // Add filter for absence of eventGranularity - only consider absolute stats + Criterion excludeTimeBucketsCriterion = + new Criterion().setField(ES_FIELD_EVENT_GRANULARITY).setCondition(Condition.IS_NULL).setValue(""); + criteria.add(excludeTimeBucketsCriterion); + filter.setOr(new ConjunctiveCriterionArray( + ImmutableList.of(new ConjunctiveCriterion().setAnd(new CriterionArray(criteria))))); + + List aspects = + timeseriesAspectService.getAspectValues(Urn.createFromString(dashboardUrn), Constants.DASHBOARD_ENTITY_NAME, + Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, maybeStartTimeMillis, maybeEndTimeMillis, maybeLimit, + null, filter); + dashboardUsageMetrics = aspects.stream().map(DashboardUsageMetricMapper::map).collect(Collectors.toList()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid resource", e); + } + return dashboardUsageMetrics; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsUtils.java new file mode 100644 index 00000000000000..958d2c707e87c3 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardUsageStatsUtils.java @@ -0,0 +1,304 @@ +package com.linkedin.datahub.graphql.resolvers.dashboard; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.DashboardUsageAggregation; +import com.linkedin.datahub.graphql.generated.DashboardUsageAggregationMetrics; +import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics; +import com.linkedin.datahub.graphql.generated.DashboardUsageQueryResultAggregations; +import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts; +import com.linkedin.datahub.graphql.generated.WindowDuration; +import com.linkedin.datahub.graphql.types.dashboard.mappers.DashboardUsageMetricMapper; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.EnvelopedAspect; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.timeseries.AggregationSpec; +import com.linkedin.timeseries.AggregationType; +import com.linkedin.timeseries.CalendarInterval; +import com.linkedin.timeseries.GenericTable; +import com.linkedin.timeseries.GroupingBucket; +import com.linkedin.timeseries.GroupingBucketType; +import com.linkedin.timeseries.TimeWindowSize; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + + +public class DashboardUsageStatsUtils { + + public static final String ES_FIELD_URN = "urn"; + public static final String ES_FIELD_TIMESTAMP = "timestampMillis"; + public static final String ES_FIELD_EVENT_GRANULARITY = "eventGranularity"; + public static final String ES_NULL_VALUE = "NULL"; + + public static List getDashboardUsageMetrics( + String dashboardUrn, + Long maybeStartTimeMillis, + Long maybeEndTimeMillis, + Integer maybeLimit, + TimeseriesAspectService timeseriesAspectService) { + List dashboardUsageMetrics; + try { + Filter filter = createUsageFilter(dashboardUrn, null, null, false); + List aspects = timeseriesAspectService.getAspectValues( + Urn.createFromString(dashboardUrn), + Constants.DASHBOARD_ENTITY_NAME, + Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, + maybeStartTimeMillis, + maybeEndTimeMillis, + maybeLimit, + null, + filter); + dashboardUsageMetrics = aspects.stream().map(DashboardUsageMetricMapper::map).collect(Collectors.toList()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid resource", e); + } + return dashboardUsageMetrics; + } + + public static DashboardUsageQueryResultAggregations getAggregations( + Filter filter, + List dailyUsageBuckets, + TimeseriesAspectService timeseriesAspectService) { + + List userUsageCounts = getUserUsageCounts(filter, timeseriesAspectService); + DashboardUsageQueryResultAggregations aggregations = new DashboardUsageQueryResultAggregations(); + aggregations.setUsers(userUsageCounts); + aggregations.setUniqueUserCount(userUsageCounts.size()); + + // Compute total viewsCount and executionsCount for queries time range from the buckets itself. + // We want to avoid issuing an additional query with a sum aggregation. + Integer totalViewsCount = null; + Integer totalExecutionsCount = null; + for (DashboardUsageAggregation bucket : dailyUsageBuckets) { + if (bucket.getMetrics().getExecutionsCount() != null) { + if (totalExecutionsCount == null) { + totalExecutionsCount = 0; + } + totalExecutionsCount += bucket.getMetrics().getExecutionsCount(); + } + if (bucket.getMetrics().getViewsCount() != null) { + if (totalViewsCount == null) { + totalViewsCount = 0; + } + totalViewsCount += bucket.getMetrics().getViewsCount(); + } + } + + aggregations.setExecutionsCount(totalExecutionsCount); + aggregations.setViewsCount(totalViewsCount); + return aggregations; + } + + public static List getBuckets( + Filter filter, + String dashboardUrn, + TimeseriesAspectService timeseriesAspectService) { + AggregationSpec usersCountAggregation = + new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("uniqueUserCount"); + AggregationSpec viewsCountAggregation = + new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("viewsCount"); + AggregationSpec executionsCountAggregation = + new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("executionsCount"); + + AggregationSpec usersCountCardinalityAggregation = + new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("uniqueUserCount"); + AggregationSpec viewsCountCardinalityAggregation = + new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("viewsCount"); + AggregationSpec executionsCountCardinalityAggregation = + new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("executionsCount"); + + AggregationSpec[] aggregationSpecs = + new AggregationSpec[]{usersCountAggregation, viewsCountAggregation, executionsCountAggregation, + usersCountCardinalityAggregation, viewsCountCardinalityAggregation, executionsCountCardinalityAggregation}; + GenericTable dailyStats = timeseriesAspectService.getAggregatedStats(Constants.DASHBOARD_ENTITY_NAME, + Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, aggregationSpecs, filter, + createUsageGroupingBuckets(CalendarInterval.DAY)); + List buckets = new ArrayList<>(); + + for (StringArray row : dailyStats.getRows()) { + DashboardUsageAggregation usageAggregation = new DashboardUsageAggregation(); + usageAggregation.setBucket(Long.valueOf(row.get(0))); + usageAggregation.setDuration(WindowDuration.DAY); + usageAggregation.setResource(dashboardUrn); + + DashboardUsageAggregationMetrics usageAggregationMetrics = new DashboardUsageAggregationMetrics(); + + if (!row.get(1).equals(ES_NULL_VALUE) && !row.get(4).equals(ES_NULL_VALUE)) { + try { + if (Integer.valueOf(row.get(4)) != 0) { + usageAggregationMetrics.setUniqueUserCount(Integer.valueOf(row.get(1))); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to convert uniqueUserCount from ES to int", e); + } + } + if (!row.get(2).equals(ES_NULL_VALUE) && !row.get(5).equals(ES_NULL_VALUE)) { + try { + if (Integer.valueOf(row.get(5)) != 0) { + usageAggregationMetrics.setViewsCount(Integer.valueOf(row.get(2))); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to convert viewsCount from ES to int", e); + } + } + if (!row.get(3).equals(ES_NULL_VALUE) && !row.get(5).equals(ES_NULL_VALUE)) { + try { + if (Integer.valueOf(row.get(6)) != 0) { + usageAggregationMetrics.setExecutionsCount(Integer.valueOf(row.get(3))); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to convert executionsCount from ES to object", e); + } + } + usageAggregation.setMetrics(usageAggregationMetrics); + buckets.add(usageAggregation); + } + return buckets; + } + + public static List getUserUsageCounts(Filter filter, TimeseriesAspectService timeseriesAspectService) { + // Sum aggregation on userCounts.count + AggregationSpec sumUsageCountsCountAggSpec = + new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.usageCount"); + AggregationSpec sumViewCountsCountAggSpec = + new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.viewsCount"); + AggregationSpec sumExecutionCountsCountAggSpec = + new AggregationSpec().setAggregationType(AggregationType.SUM).setFieldPath("userCounts.executionsCount"); + + AggregationSpec usageCountsCardinalityAggSpec = + new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("userCounts.usageCount"); + AggregationSpec viewCountsCardinalityAggSpec = + new AggregationSpec().setAggregationType(AggregationType.CARDINALITY).setFieldPath("userCounts.viewsCount"); + AggregationSpec executionCountsCardinalityAggSpec = + new AggregationSpec().setAggregationType(AggregationType.CARDINALITY) + .setFieldPath("userCounts.executionsCount"); + AggregationSpec[] aggregationSpecs = + new AggregationSpec[]{sumUsageCountsCountAggSpec, sumViewCountsCountAggSpec, sumExecutionCountsCountAggSpec, + usageCountsCardinalityAggSpec, viewCountsCardinalityAggSpec, executionCountsCardinalityAggSpec}; + + // String grouping bucket on userCounts.user + GroupingBucket userGroupingBucket = + new GroupingBucket().setKey("userCounts.user").setType(GroupingBucketType.STRING_GROUPING_BUCKET); + GroupingBucket[] groupingBuckets = new GroupingBucket[]{userGroupingBucket}; + + // Query backend + GenericTable result = timeseriesAspectService.getAggregatedStats(Constants.DASHBOARD_ENTITY_NAME, + Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME, aggregationSpecs, filter, groupingBuckets); + // Process response + List userUsageCounts = new ArrayList<>(); + for (StringArray row : result.getRows()) { + DashboardUserUsageCounts userUsageCount = new DashboardUserUsageCounts(); + + CorpUser partialUser = new CorpUser(); + partialUser.setUrn(row.get(0)); + userUsageCount.setUser(partialUser); + + if (!row.get(1).equals(ES_NULL_VALUE) && !row.get(4).equals(ES_NULL_VALUE)) { + try { + if (Integer.valueOf(row.get(4)) != 0) { + userUsageCount.setUsageCount(Integer.valueOf(row.get(1))); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to convert user usage count from ES to int", e); + } + } + if (!row.get(2).equals(ES_NULL_VALUE) && row.get(5).equals(ES_NULL_VALUE)) { + try { + if (Integer.valueOf(row.get(5)) != 0) { + userUsageCount.setViewsCount(Integer.valueOf(row.get(2))); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to convert user views count from ES to int", e); + } + } + if (!row.get(3).equals(ES_NULL_VALUE) && !row.get(6).equals(ES_NULL_VALUE)) { + try { + if (Integer.valueOf(row.get(6)) != 0) { + userUsageCount.setExecutionsCount(Integer.valueOf(row.get(3))); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to convert user executions count from ES to int", e); + } + } + userUsageCounts.add(userUsageCount); + } + + // Sort in descending order + userUsageCounts.sort((a, b) -> (b.getUsageCount() - a.getUsageCount())); + return userUsageCounts; + } + + private static GroupingBucket[] createUsageGroupingBuckets(CalendarInterval calenderInterval) { + GroupingBucket timestampBucket = new GroupingBucket(); + timestampBucket.setKey(ES_FIELD_TIMESTAMP) + .setType(GroupingBucketType.DATE_GROUPING_BUCKET) + .setTimeWindowSize(new TimeWindowSize().setMultiple(1).setUnit(calenderInterval)); + return new GroupingBucket[]{timestampBucket}; + } + + public static Filter createUsageFilter( + String dashboardUrn, + Long startTime, + Long endTime, + boolean byBucket) { + Filter filter = new Filter(); + final ArrayList criteria = new ArrayList<>(); + + // Add filter for urn == dashboardUrn + Criterion dashboardUrnCriterion = + new Criterion().setField(ES_FIELD_URN).setCondition(Condition.EQUAL).setValue(dashboardUrn); + criteria.add(dashboardUrnCriterion); + + if (startTime != null) { + // Add filter for start time + Criterion startTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP) + .setCondition(Condition.GREATER_THAN_OR_EQUAL_TO) + .setValue(Long.toString(startTime)); + criteria.add(startTimeCriterion); + } + + if (endTime != null) { + // Add filter for end time + Criterion endTimeCriterion = new Criterion().setField(ES_FIELD_TIMESTAMP) + .setCondition(Condition.LESS_THAN_OR_EQUAL_TO) + .setValue(Long.toString(endTime)); + criteria.add(endTimeCriterion); + } + + if (byBucket) { + // Add filter for presence of eventGranularity - only consider bucket stats and not absolute stats + // since unit is mandatory, we assume if eventGranularity contains unit, then it is not null + Criterion onlyTimeBucketsCriterion = + new Criterion().setField(ES_FIELD_EVENT_GRANULARITY).setCondition(Condition.CONTAIN).setValue("unit"); + criteria.add(onlyTimeBucketsCriterion); + } else { + // Add filter for absence of eventGranularity - only consider absolute stats + Criterion excludeTimeBucketsCriterion = + new Criterion().setField(ES_FIELD_EVENT_GRANULARITY).setCondition(Condition.IS_NULL).setValue(""); + criteria.add(excludeTimeBucketsCriterion); + } + + filter.setOr(new ConjunctiveCriterionArray( + ImmutableList.of(new ConjunctiveCriterion().setAnd(new CriterionArray(criteria))))); + return filter; + } + + + public static Long timeMinusOneMonth(long time) { + final long oneHourMillis = 60 * 60 * 1000; + final long oneDayMillis = 24 * oneHourMillis; + return time - (31 * oneDayMillis + 1); + } + + private DashboardUsageStatsUtils() { } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java index 4d12bda18057a1..729446eab1fa9e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolver.java @@ -10,6 +10,7 @@ import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.Health; import com.linkedin.datahub.graphql.generated.HealthStatus; +import com.linkedin.datahub.graphql.generated.HealthStatusType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.query.filter.Condition; @@ -28,7 +29,6 @@ import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -36,6 +36,8 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; /** @@ -45,34 +47,45 @@ * health status will be undefined for the Dataset. * */ -public class DatasetHealthResolver implements DataFetcher> { +@Slf4j +public class DatasetHealthResolver implements DataFetcher>> { private static final String ASSERTS_RELATIONSHIP_NAME = "Asserts"; private static final String ASSERTION_RUN_EVENT_SUCCESS_TYPE = "SUCCESS"; - private static final CachedHealth NO_HEALTH = new CachedHealth(false, null); private final GraphClient _graphClient; private final TimeseriesAspectService _timeseriesAspectService; + private final Config _config; private final Cache _statusCache; - public DatasetHealthResolver(final GraphClient graphClient, final TimeseriesAspectService timeseriesAspectService) { + public DatasetHealthResolver( + final GraphClient graphClient, + final TimeseriesAspectService timeseriesAspectService) { + this(graphClient, timeseriesAspectService, new Config(true)); + + } + public DatasetHealthResolver( + final GraphClient graphClient, + final TimeseriesAspectService timeseriesAspectService, + final Config config) { _graphClient = graphClient; _timeseriesAspectService = timeseriesAspectService; _statusCache = CacheBuilder.newBuilder() .maximumSize(10000) .expireAfterWrite(1, TimeUnit.MINUTES) .build(); + _config = config; } @Override - public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + public CompletableFuture> get(final DataFetchingEnvironment environment) throws Exception { final Dataset parent = environment.getSource(); return CompletableFuture.supplyAsync(() -> { try { final CachedHealth cachedStatus = _statusCache.get(parent.getUrn(), () -> ( computeHealthStatusForDataset(parent.getUrn(), environment.getContext()))); - return cachedStatus.hasStatus ? cachedStatus.health : null; + return cachedStatus.healths; } catch (Exception e) { throw new RuntimeException("Failed to resolve dataset's health status.", e); } @@ -88,11 +101,15 @@ public CompletableFuture get(final DataFetchingEnvironment environment) * */ private CachedHealth computeHealthStatusForDataset(final String datasetUrn, final QueryContext context) { - final Health result = computeAssertionHealthForDataset(datasetUrn, context); - if (result == null) { - return NO_HEALTH; + final List healthStatuses = new ArrayList<>(); + + if (_config.getAssertionsEnabled()) { + final Health assertionsHealth = computeAssertionHealthForDataset(datasetUrn, context); + if (assertionsHealth != null) { + healthStatuses.add(assertionsHealth); + } } - return new CachedHealth(true, result); + return new CachedHealth(healthStatuses); } /** @@ -122,10 +139,18 @@ private Health computeAssertionHealthForDataset(final String datasetUrn, final Q .stream() .map(relationship -> relationship.getEntity().toString()).collect(Collectors.toSet()); - final List failingAssertionUrns = getFailingAssertionUrns(datasetUrn, activeAssertionUrns); + final GenericTable assertionRunResults = getAssertionRunsTable(datasetUrn); + + if (!assertionRunResults.hasRows() || assertionRunResults.getRows().size() == 0) { + // No assertion run results found. Return empty health! + return null; + } + + final List failingAssertionUrns = getFailingAssertionUrns(assertionRunResults, activeAssertionUrns); // Finally compute & return the health. final Health health = new Health(); + health.setType(HealthStatusType.ASSERTIONS); if (failingAssertionUrns.size() > 0) { health.setStatus(HealthStatus.FAIL); health.setMessage(String.format("Dataset is failing %s/%s assertions.", failingAssertionUrns.size(), @@ -141,20 +166,18 @@ private Health computeAssertionHealthForDataset(final String datasetUrn, final Q return null; } - private List getFailingAssertionUrns(final String asserteeUrn, final Set candidateAssertionUrns) { - // Query timeseries backend - GenericTable result = _timeseriesAspectService.getAggregatedStats( + private GenericTable getAssertionRunsTable(final String asserteeUrn) { + return _timeseriesAspectService.getAggregatedStats( Constants.ASSERTION_ENTITY_NAME, Constants.ASSERTION_RUN_EVENT_ASPECT_NAME, createAssertionAggregationSpecs(), createAssertionsFilter(asserteeUrn), createAssertionGroupingBuckets()); - if (!result.hasRows()) { - // No completed assertion runs found. Return empty list. - return Collections.emptyList(); - } + } + + private List getFailingAssertionUrns(final GenericTable assertionRunsResult, final Set candidateAssertionUrns) { // Create the buckets based on the result - return resultToFailedAssertionUrns(result.getRows(), candidateAssertionUrns); + return resultToFailedAssertionUrns(assertionRunsResult.getRows(), candidateAssertionUrns); } private Filter createAssertionsFilter(final String datasetUrn) { @@ -213,9 +236,14 @@ private List resultToFailedAssertionUrns(final StringArrayArray rows, fi return failedAssertionUrns; } - @AllArgsConstructor + @Data + @AllArgsConstructor + public static class Config { + private Boolean assertionsEnabled; + } + + @AllArgsConstructor private static class CachedHealth { - private final boolean hasStatus; - private final Health health; + private final List healths; } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java new file mode 100644 index 00000000000000..f27fd604a746f6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java @@ -0,0 +1,93 @@ +package com.linkedin.datahub.graphql.resolvers.dataset; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.usage.UsageClient; +import com.linkedin.usage.UsageTimeRange; +import com.linkedin.usage.UserUsageCounts; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + + +/** + * This resolver is a thin wrapper around the {@link DatasetUsageStatsResolver} which simply + * computes some aggregate usage metrics for a Dashboard. + */ +@Slf4j +public class DatasetStatsSummaryResolver implements DataFetcher> { + + // The maximum number of top users to show in the summary stats + private static final Integer MAX_TOP_USERS = 5; + + private final UsageClient usageClient; + private final Cache summaryCache; + + public DatasetStatsSummaryResolver(final UsageClient usageClient) { + this.usageClient = usageClient; + this.summaryCache = CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterWrite(6, TimeUnit.HOURS) // TODO: Make caching duration configurable externally. + .build(); + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); + + return CompletableFuture.supplyAsync(() -> { + + if (this.summaryCache.getIfPresent(resourceUrn) != null) { + return this.summaryCache.getIfPresent(resourceUrn); + } + + try { + + com.linkedin.usage.UsageQueryResult + usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), UsageTimeRange.MONTH, context.getAuthentication()); + + final DatasetStatsSummary result = new DatasetStatsSummary(); + result.setQueryCountLast30Days(usageQueryResult.getAggregations().getTotalSqlQueries()); + result.setUniqueUserCountLast30Days(usageQueryResult.getAggregations().getUniqueUserCount()); + if (usageQueryResult.getAggregations().hasUsers()) { + result.setTopUsersLast30Days(trimUsers(usageQueryResult.getAggregations().getUsers() + .stream() + .filter(UserUsageCounts::hasUser) + .sorted((a, b) -> (b.getCount() - a.getCount())) + .map(userCounts -> createPartialUser(Objects.requireNonNull(userCounts.getUser()))) + .collect(Collectors.toList()))); + } + this.summaryCache.put(resourceUrn, result); + return result; + } catch (Exception e) { + log.error(String.format("Failed to load Usage Stats summary for resource %s", resourceUrn.toString()), e); + return null; // Do not throw when loading usage summary fails. + } + }); + } + + private List trimUsers(final List originalUsers) { + if (originalUsers.size() > MAX_TOP_USERS) { + return originalUsers.subList(0, MAX_TOP_USERS); + } + return originalUsers; + } + + private CorpUser createPartialUser(final Urn userUrn) { + final CorpUser result = new CorpUser(); + result.setUrn(userUrn.toString()); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java new file mode 100644 index 00000000000000..0476963b92e9a1 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java @@ -0,0 +1,60 @@ +package com.linkedin.datahub.graphql.resolvers.dataset; + +import com.datahub.authorization.ResourceSpec; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.UsageQueryResult; +import com.linkedin.datahub.graphql.types.usage.UsageQueryResultMapper; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.usage.UsageClient; +import com.linkedin.usage.UsageTimeRange; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class DatasetUsageStatsResolver implements DataFetcher> { + + private final UsageClient usageClient; + + public DatasetUsageStatsResolver(final UsageClient usageClient) { + this.usageClient = usageClient; + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); + final UsageTimeRange range = UsageTimeRange.valueOf(environment.getArgument("range")); + + return CompletableFuture.supplyAsync(() -> { + if (!isAuthorized(resourceUrn, context)) { + log.debug("User {} is not authorized to view usage information for dataset {}", + context.getActorUrn(), + resourceUrn.toString()); + return null; + } + try { + com.linkedin.usage.UsageQueryResult + usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), range, context.getAuthentication()); + return UsageQueryResultMapper.map(usageQueryResult); + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", resourceUrn.toString()), e); + } + }); + } + + private boolean isAuthorized(final Urn resourceUrn, final QueryContext context) { + return AuthorizationUtils.isAuthorized(context, + Optional.of(new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString())), + PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 3883a4b6f57693..eb53c5926af324 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -1,19 +1,16 @@ package com.linkedin.datahub.graphql.resolvers.domain; -import com.google.common.collect.ImmutableList; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; -import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; -import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.CreateDomainInput; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.key.DomainKey; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; @@ -26,7 +23,7 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; /** - * Resolver used for creating a new Domain on DataHub. Requires the MANAGE_DOMAINS privilege. + * Resolver used for creating a new Domain on DataHub. Requires the CREATE_DOMAINS or MANAGE_DOMAINS privilege. */ @Slf4j @RequiredArgsConstructor @@ -42,12 +39,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws return CompletableFuture.supplyAsync(() -> { - if (!isAuthorizedToCreateDomain(context)) { + if (!AuthorizationUtils.canCreateDomains(context)) { throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } - // TODO: Add exists check. Currently this can override previously created domains. - try { // Create the Domain Key final DomainKey key = new DomainKey(); @@ -56,6 +51,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setId(id); + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.DOMAIN_ENTITY_NAME), context.getAuthentication())) { + throw new IllegalArgumentException("This Domain already exists!"); + } + // Create the MCP final MetadataChangeProposal proposal = new MetadataChangeProposal(); proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); @@ -63,6 +62,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); proposal.setAspect(GenericRecordUtils.serializeAspect(mapDomainProperties(input))); proposal.setChangeType(ChangeType.UPSERT); + return _entityClient.ingestProposal(proposal, context.getAuthentication()); } catch (Exception e) { log.error("Failed to create Domain with id: {}, name: {}: {}", input.getId(), input.getName(), e.getMessage()); @@ -77,15 +77,4 @@ private DomainProperties mapDomainProperties(final CreateDomainInput input) { result.setDescription(input.getDescription(), SetMode.IGNORE_NULL); return result; } - - private boolean isAuthorizedToCreateDomain(final QueryContext context) { - final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of( - new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType())) - )); - - return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - orPrivilegeGroups); - } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java new file mode 100644 index 00000000000000..ba712f8e6c7494 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; + + +/** + * Resolver responsible for hard deleting a particular DataHub Corp Group + */ +@Slf4j +public class DeleteDomainResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public DeleteDomainResolver(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final String domainUrn = environment.getArgument("urn"); + final Urn urn = Urn.createFromString(domainUrn); + return CompletableFuture.supplyAsync(() -> { + + if (AuthorizationUtils.canManageDomains(context) || AuthorizationUtils.canDeleteEntity(urn, context)) { + try { + _entityClient.deleteEntity(urn, context.getAuthentication()); + log.info(String.format("I've successfully deleted the entity %s with urn", domainUrn)); + + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync(() -> { + try { + _entityClient.deleteEntityReferences(urn, context.getAuthentication()); + } catch (RemoteInvocationException e) { + log.error(String.format("Caught exception while attempting to clear all entity references for Domain with urn %s", urn), e); + } + }); + + return true; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to perform delete against domain with urn %s", domainUrn), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index 076387449816ac..0cf6d70e6909ec 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -45,7 +45,7 @@ public CompletableFuture get(final DataFetchingEnvironment en return CompletableFuture.supplyAsync(() -> { - if (AuthorizationUtils.canManageDomains(context)) { + if (AuthorizationUtils.canCreateDomains(context)) { final ListDomainsInput input = bindArgument(environment.getArgument("input"), ListDomainsInput.class); final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolver.java new file mode 100644 index 00000000000000..c440f1a7721dcc --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolver.java @@ -0,0 +1,122 @@ +package com.linkedin.datahub.graphql.resolvers.glossary; + +import com.linkedin.common.GlossaryTermUrnArray; +import com.linkedin.common.urn.GlossaryTermUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.RelatedTermsInput; +import com.linkedin.datahub.graphql.generated.TermRelationshipType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.glossary.GlossaryRelatedTerms; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + +@Slf4j +@RequiredArgsConstructor +public class AddRelatedTermsResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final RelatedTermsInput input = bindArgument(environment.getArgument("input"), RelatedTermsInput.class); + + return CompletableFuture.supplyAsync(() -> { + if (AuthorizationUtils.canManageGlossaries(context)) { + try { + final TermRelationshipType relationshipType = input.getRelationshipType(); + final Urn urn = Urn.createFromString(input.getUrn()); + final List termUrns = input.getTermUrns().stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + validateRelatedTermsInput(urn, termUrns); + Urn actor = Urn.createFromString(((QueryContext) context).getActorUrn()); + + GlossaryRelatedTerms glossaryRelatedTerms = (GlossaryRelatedTerms) getAspectFromEntity( + urn.toString(), + Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME, + _entityService, + null + ); + if (glossaryRelatedTerms == null) { + glossaryRelatedTerms = new GlossaryRelatedTerms(); + } + + if (relationshipType == TermRelationshipType.isA) { + if (!glossaryRelatedTerms.hasIsRelatedTerms()) { + glossaryRelatedTerms.setIsRelatedTerms(new GlossaryTermUrnArray()); + } + final GlossaryTermUrnArray existingTermUrns = glossaryRelatedTerms.getIsRelatedTerms(); + + return updateRelatedTerms(termUrns, existingTermUrns, urn, glossaryRelatedTerms, actor); + } else { + if (!glossaryRelatedTerms.hasHasRelatedTerms()) { + glossaryRelatedTerms.setHasRelatedTerms(new GlossaryTermUrnArray()); + } + final GlossaryTermUrnArray existingTermUrns = glossaryRelatedTerms.getHasRelatedTerms(); + + return updateRelatedTerms(termUrns, existingTermUrns, urn, glossaryRelatedTerms, actor); + } + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to add related terms to %s", input.getUrn()), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } + + public Boolean validateRelatedTermsInput(Urn urn, List termUrns) { + if (!urn.getEntityType().equals(Constants.GLOSSARY_TERM_ENTITY_NAME) || !_entityService.exists(urn)) { + throw new IllegalArgumentException(String.format("Failed to update %s. %s either does not exist or is not a glossaryTerm.", urn, urn)); + } + + for (Urn termUrn : termUrns) { + if (termUrn.equals(urn)) { + throw new IllegalArgumentException(String.format("Failed to update %s. Tried to create related term with itself.", urn)); + } else if (!termUrn.getEntityType().equals(Constants.GLOSSARY_TERM_ENTITY_NAME)) { + throw new IllegalArgumentException(String.format("Failed to update %s. %s is not a glossaryTerm.", urn, termUrn)); + } else if (!_entityService.exists(termUrn)) { + throw new IllegalArgumentException(String.format("Failed to update %s. %s does not exist.", urn, termUrn)); + } + } + return true; + } + + private Boolean updateRelatedTerms(List termUrns, GlossaryTermUrnArray existingTermUrns, Urn urn, GlossaryRelatedTerms glossaryRelatedTerms, Urn actor) { + List termsToAdd = new ArrayList<>(); + for (Urn termUrn : termUrns) { + if (existingTermUrns.stream().anyMatch(association -> association.equals(termUrn))) { + continue; + } + termsToAdd.add(termUrn); + } + + if (termsToAdd.size() == 0) { + return true; + } + + for (Urn termUrn : termsToAdd) { + GlossaryTermUrn newUrn = new GlossaryTermUrn(termUrn.getId()); + + existingTermUrns.add(newUrn); + } + persistAspect(urn, Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME, glossaryRelatedTerms, actor, _entityService); + return true; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java index aab3b4ed862d6f..04b752ff5e7d68 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java @@ -11,6 +11,7 @@ import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryNodeKey; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; @@ -44,6 +45,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setName(id); + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.GLOSSARY_NODE_ENTITY_NAME), context.getAuthentication())) { + throw new IllegalArgumentException("This Glossary Node already exists!"); + } + final MetadataChangeProposal proposal = new MetadataChangeProposal(); proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); proposal.setEntityType(Constants.GLOSSARY_NODE_ENTITY_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java index 3b72b6a4fa5a2d..e40d159f99969e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryTermResolver.java @@ -11,6 +11,7 @@ import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryTermKey; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; @@ -44,6 +45,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); key.setName(id); + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.GLOSSARY_TERM_ENTITY_NAME), context.getAuthentication())) { + throw new IllegalArgumentException("This Glossary Term already exists!"); + } + final MetadataChangeProposal proposal = new MetadataChangeProposal(); proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); proposal.setEntityType(Constants.GLOSSARY_TERM_ENTITY_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolver.java index 6bdb9d1ccd1f69..4aa5e6ff9a8d92 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/DeleteGlossaryEntityResolver.java @@ -6,11 +6,14 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class DeleteGlossaryEntityResolver implements DataFetcher> { private final EntityClient _entityClient; @@ -34,6 +37,16 @@ public CompletableFuture get(final DataFetchingEnvironment environment) try { _entityClient.deleteEntity(entityUrn, context.getAuthentication()); + + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync(() -> { + try { + _entityClient.deleteEntityReferences(entityUrn, context.getAuthentication()); + } catch (RemoteInvocationException e) { + log.error(String.format("Caught exception while attempting to clear all entity references for glossary entity with urn %s", entityUrn), e); + } + }); + return true; } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform delete against glossary entity with urn %s", entityUrn), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolver.java new file mode 100644 index 00000000000000..2ca9e081d7f3ff --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolver.java @@ -0,0 +1,90 @@ +package com.linkedin.datahub.graphql.resolvers.glossary; + +import com.linkedin.common.GlossaryTermUrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.RelatedTermsInput; +import com.linkedin.datahub.graphql.generated.TermRelationshipType; +import com.linkedin.glossary.GlossaryRelatedTerms; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.getAspectFromEntity; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.persistAspect; + +@Slf4j +@RequiredArgsConstructor +public class RemoveRelatedTermsResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final RelatedTermsInput input = bindArgument(environment.getArgument("input"), RelatedTermsInput.class); + + return CompletableFuture.supplyAsync(() -> { + if (AuthorizationUtils.canManageGlossaries(context)) { + try { + final TermRelationshipType relationshipType = input.getRelationshipType(); + final Urn urn = Urn.createFromString(input.getUrn()); + final List termUrnsToRemove = input.getTermUrns().stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + + if (!urn.getEntityType().equals(Constants.GLOSSARY_TERM_ENTITY_NAME) || !_entityService.exists(urn)) { + throw new IllegalArgumentException(String.format("Failed to update %s. %s either does not exist or is not a glossaryTerm.", urn, urn)); + } + + Urn actor = Urn.createFromString(((QueryContext) context).getActorUrn()); + + GlossaryRelatedTerms glossaryRelatedTerms = (GlossaryRelatedTerms) getAspectFromEntity( + urn.toString(), + Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME, + _entityService, + null + ); + if (glossaryRelatedTerms == null) { + throw new RuntimeException(String.format("Related Terms for this Urn do not exist: %s", urn)); + } + + if (relationshipType == TermRelationshipType.isA) { + if (!glossaryRelatedTerms.hasIsRelatedTerms()) { + throw new RuntimeException("Failed to remove from GlossaryRelatedTerms as they do not exist for this Glossary Term"); + } + final GlossaryTermUrnArray existingTermUrns = glossaryRelatedTerms.getIsRelatedTerms(); + + existingTermUrns.removeIf(termUrn -> termUrnsToRemove.stream().anyMatch(termUrn::equals)); + persistAspect(urn, Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME, glossaryRelatedTerms, actor, _entityService); + return true; + } else { + if (!glossaryRelatedTerms.hasHasRelatedTerms()) { + throw new RuntimeException("Failed to remove from GlossaryRelatedTerms as they do not exist for this Glossary Term"); + } + final GlossaryTermUrnArray existingTermUrns = glossaryRelatedTerms.getHasRelatedTerms(); + + existingTermUrns.removeIf(termUrn -> termUrnsToRemove.stream().anyMatch(termUrn::equals)); + persistAspect(urn, Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME, glossaryRelatedTerms, actor, _entityService); + return true; + } + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to removes related terms from %s", input.getUrn()), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolver.java index 349e5c46ee0cd2..daff0962bc2e81 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolver.java @@ -1,148 +1,81 @@ package com.linkedin.datahub.graphql.resolvers.group; -import com.google.common.collect.ImmutableList; -import com.linkedin.common.UrnArray; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.Origin; +import com.linkedin.common.OriginType; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; -import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; -import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.AddGroupMembersInput; -import com.linkedin.entity.EntityResponse; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.identity.GroupMembership; -import com.linkedin.metadata.authorization.PoliciesConfig; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; -import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; -import static com.linkedin.datahub.graphql.resolvers.AuthUtils.*; +import static com.linkedin.datahub.graphql.authorization.AuthorizationUtils.*; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; -import static com.linkedin.metadata.Constants.*; /** - * Resolver that adds a set of members to a group, if the user and group both exist. + * Resolver that adds a set of native members to a group, if the user and group both exist. */ public class AddGroupMembersResolver implements DataFetcher> { - private final EntityClient _entityClient; + private final GroupService _groupService; - public AddGroupMembersResolver(final EntityClient entityClient) { - _entityClient = entityClient; + public AddGroupMembersResolver(final GroupService groupService) { + _groupService = groupService; } @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final AddGroupMembersInput input = bindArgument(environment.getArgument("input"), AddGroupMembersInput.class); + final String groupUrnStr = input.getGroupUrn(); final QueryContext context = environment.getContext(); + final Authentication authentication = context.getAuthentication(); + Urn groupUrn = Urn.createFromString(groupUrnStr); - if (isAuthorized(input, context)) { - final String groupUrnStr = input.getGroupUrn(); - final List userUrnStrs = input.getUserUrns(); - - return CompletableFuture.runAsync(() -> { - if (!groupExists(groupUrnStr, context)) { - // The group doesn't exist. - throw new DataHubGraphQLException("Failed to add member to group. Group does not exist.", DataHubGraphQLErrorCode.NOT_FOUND); - } - }) - .thenApply(ignored -> CompletableFuture.allOf( - userUrnStrs.stream().map(userUrnStr -> CompletableFuture.runAsync(() -> - addUserToGroup(userUrnStr, groupUrnStr, context) - )).toArray(CompletableFuture[]::new))) - .thenApply((ignored) -> Boolean.TRUE); + if (!canEditGroupMembers(groupUrnStr, context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); } - throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); - } - - private boolean isAuthorized(AddGroupMembersInput input, QueryContext context) { - final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of( - ALL_PRIVILEGES_GROUP, - new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.EDIT_GROUP_MEMBERS_PRIVILEGE.getType())) - )); - return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - CORP_GROUP_ENTITY_NAME, - input.getGroupUrn(), - orPrivilegeGroups); - } - - private void addUserToGroup(final String userUrnStr, final String groupUrnStr, final QueryContext context) { - try { - // First, fetch user's group membership aspect. - Urn userUrn = Urn.createFromString(userUrnStr); - final EntityResponse entityResponse = - _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, Collections.singleton(userUrn), - Collections.singleton(GROUP_MEMBERSHIP_ASPECT_NAME), context.getAuthentication()).get(userUrn); - - GroupMembership groupMembership; - if (entityResponse == null || !entityResponse.getAspects().containsKey(GROUP_MEMBERSHIP_ASPECT_NAME)) { - // Verify the user exists - if (!userExists(userUrnStr, context)) { - throw new DataHubGraphQLException("Failed to add member to group. User does not exist.", DataHubGraphQLErrorCode.NOT_FOUND); + if (!_groupService.groupExists(groupUrn)) { + // The group doesn't exist. + throw new DataHubGraphQLException( + String.format("Failed to add members to group %s. Group does not exist.", groupUrnStr), + DataHubGraphQLErrorCode.NOT_FOUND); + } + return CompletableFuture.supplyAsync(() -> { + Origin groupOrigin = _groupService.getGroupOrigin(groupUrn); + if (groupOrigin == null || !groupOrigin.hasType()) { + try { + _groupService.migrateGroupMembershipToNativeGroupMembership(groupUrn, context.getActorUrn(), + context.getAuthentication()); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to migrate group membership for group %s when adding group members", groupUrnStr)); } - // If the user doesn't have one, create one. - groupMembership = new GroupMembership(); - groupMembership.setGroups(new UrnArray()); - } else { - groupMembership = new GroupMembership(entityResponse.getAspects() - .get(GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()); + } else if (groupOrigin.getType() == OriginType.EXTERNAL) { + throw new RuntimeException(String.format( + "Group %s was ingested from an external provider and cannot have members manually added to it", + groupUrnStr)); } - // Handle the duplicate case. - final Urn groupUrn = Urn.createFromString(groupUrnStr); - groupMembership.getGroups().remove(groupUrn); - groupMembership.getGroups().add(groupUrn); - // Finally, create the MetadataChangeProposal. - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(Urn.createFromString(userUrnStr)); - proposal.setEntityType(CORP_USER_ENTITY_NAME); - proposal.setAspectName(GROUP_MEMBERSHIP_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(groupMembership)); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); - } catch (Exception e) { - throw new RuntimeException("Failed to add member to group", e); - } - } - - private boolean groupExists(final String groupUrnStr, final QueryContext context) { - try { - Urn groupUrn = Urn.createFromString(groupUrnStr); - final EntityResponse entityResponse = _entityClient.batchGetV2( - CORP_GROUP_ENTITY_NAME, - Collections.singleton(groupUrn), - Collections.singleton(CORP_GROUP_KEY_ASPECT_NAME), - context.getAuthentication()).get(groupUrn); - return entityResponse != null && entityResponse.getAspects().containsKey(CORP_GROUP_KEY_ASPECT_NAME); - } catch (Exception e) { - throw new DataHubGraphQLException("Failed to fetch group!", DataHubGraphQLErrorCode.SERVER_ERROR); - } - } - - private boolean userExists(final String userUrnStr, final QueryContext context) { - try { - Urn userUrn = Urn.createFromString(userUrnStr); - final EntityResponse entityResponse = _entityClient.batchGetV2( - CORP_USER_ENTITY_NAME, - Collections.singleton(userUrn), - Collections.singleton(CORP_USER_KEY_ASPECT_NAME), - context.getAuthentication()).get(userUrn); - return entityResponse != null && entityResponse.getAspects().containsKey(CORP_USER_KEY_ASPECT_NAME); - } catch (Exception e) { - throw new DataHubGraphQLException("Failed to fetch user!", DataHubGraphQLErrorCode.SERVER_ERROR); - } + try { + // Add each user to the group + final List userUrnList = input.getUserUrns().stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + userUrnList.forEach(userUrn -> _groupService.addUserToNativeGroup(userUrn, groupUrn, authentication)); + return true; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to add group members to group %s", groupUrnStr)); + } + }); } -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolver.java index 3b8fec4e2ba81e..75f2a61287ecc4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolver.java @@ -1,18 +1,12 @@ package com.linkedin.datahub.graphql.resolvers.group; -import com.linkedin.common.CorpGroupUrnArray; -import com.linkedin.common.CorpuserUrnArray; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.CreateGroupInput; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.identity.CorpGroupInfo; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.CorpGroupKey; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.UUID; @@ -24,49 +18,34 @@ // Currently, this resolver will override the group details, but not group membership, if a group with the same name already exists. public class CreateGroupResolver implements DataFetcher> { - private final EntityClient _entityClient; + private final GroupService _groupService; - public CreateGroupResolver(final EntityClient entityClient) { - _entityClient = entityClient; + public CreateGroupResolver(final GroupService groupService) { + _groupService = groupService; } @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { - final QueryContext context = environment.getContext(); + Authentication authentication = context.getAuthentication(); - if (AuthorizationUtils.canManageUsersAndGroups(context)) { - final CreateGroupInput input = bindArgument(environment.getArgument("input"), CreateGroupInput.class); - - return CompletableFuture.supplyAsync(() -> { - try { - // First, check if the group already exists. - // Create the Group key. - final CorpGroupKey key = new CorpGroupKey(); - final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); - key.setName(id); // 'name' in the key really reflects nothing more than a stable "id". - - // Create the Group info. - final CorpGroupInfo info = new CorpGroupInfo(); - info.setDisplayName(input.getName()); - info.setDescription(input.getDescription()); - info.setGroups(new CorpGroupUrnArray()); - info.setMembers(new CorpuserUrnArray()); - info.setAdmins(new CorpuserUrnArray()); - - // Finally, create the MetadataChangeProposal. - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - proposal.setEntityType(Constants.CORP_GROUP_ENTITY_NAME); - proposal.setAspectName(Constants.CORP_GROUP_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); - return _entityClient.ingestProposal(proposal, context.getAuthentication()); - } catch (Exception e) { - throw new RuntimeException("Failed to create group", e); - } - }); + if (!AuthorizationUtils.canManageUsersAndGroups(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); } - throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + final CreateGroupInput input = bindArgument(environment.getArgument("input"), CreateGroupInput.class); + + return CompletableFuture.supplyAsync(() -> { + try { + // First, check if the group already exists. + // Create the Group key. + final CorpGroupKey key = new CorpGroupKey(); + final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + key.setName(id); // 'name' in the key really reflects nothing more than a stable "id". + return _groupService.createNativeGroup(key, input.getName(), input.getDescription(), authentication); + } catch (Exception e) { + throw new RuntimeException("Failed to create group", e); + } + }); } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolver.java index a312cdfd48efe0..287b4aa7b5dbd7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolver.java @@ -1,94 +1,79 @@ package com.linkedin.datahub.graphql.resolvers.group; -import com.google.common.collect.ImmutableList; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.Origin; +import com.linkedin.common.OriginType; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; -import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; -import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.RemoveGroupMembersInput; -import com.linkedin.entity.EntityResponse; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.identity.GroupMembership; -import com.linkedin.metadata.authorization.PoliciesConfig; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; -import java.util.Collections; -import java.util.Map; -import java.util.Set; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import static com.linkedin.datahub.graphql.resolvers.AuthUtils.*; +import static com.linkedin.datahub.graphql.authorization.AuthorizationUtils.*; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; -import static com.linkedin.metadata.Constants.*; public class RemoveGroupMembersResolver implements DataFetcher> { - private final EntityClient _entityClient; + private final GroupService _groupService; - public RemoveGroupMembersResolver(final EntityClient entityClient) { - _entityClient = entityClient; + public RemoveGroupMembersResolver(final GroupService groupService) { + _groupService = groupService; } @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final RemoveGroupMembersInput input = bindArgument(environment.getArgument("input"), RemoveGroupMembersInput.class); + final String groupUrnStr = input.getGroupUrn(); final QueryContext context = environment.getContext(); + final Authentication authentication = context.getAuthentication(); - if (isAuthorized(input, context)) { - final Urn groupUrn = Urn.createFromString(input.getGroupUrn()); - final Set userUrns = input.getUserUrns().stream().map(UrnUtils::getUrn).collect(Collectors.toSet()); - final Map entityResponseMap = _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, - userUrns, Collections.singleton(GROUP_MEMBERSHIP_ASPECT_NAME), context.getAuthentication()); - return CompletableFuture.allOf(userUrns.stream().map(userUrn -> CompletableFuture.supplyAsync(() -> { - try { - EntityResponse entityResponse = entityResponseMap.get(userUrn); - if (entityResponse == null) { - return false; - } + if (!canEditGroupMembers(groupUrnStr, context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } - final GroupMembership groupMembership = - new GroupMembership(entityResponse.getAspects().get(GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()); - if (groupMembership.getGroups().remove(groupUrn)) { - // Finally, create the MetadataChangeProposal. - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(userUrn); - proposal.setEntityType(CORP_USER_ENTITY_NAME); - proposal.setAspectName(GROUP_MEMBERSHIP_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(groupMembership)); - proposal.setChangeType(ChangeType.UPSERT); - _entityClient.ingestProposal(proposal, context.getAuthentication()); - return true; - } + final Urn groupUrn = Urn.createFromString(groupUrnStr); + final List userUrnList = input.getUserUrns().stream().map(UrnUtils::getUrn).collect(Collectors.toList()); - return true; - } catch (Exception e) { - throw new RuntimeException("Failed to remove member from group", e); - } - })).toArray(CompletableFuture[]::new)).thenApply(ignored -> Boolean.TRUE); + if (!_groupService.groupExists(groupUrn)) { + // The group doesn't exist. + throw new DataHubGraphQLException( + String.format("Failed to add remove members from group %s. Group does not exist.", groupUrnStr), + DataHubGraphQLErrorCode.NOT_FOUND); } - throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); - } - private boolean isAuthorized(RemoveGroupMembersInput input, QueryContext context) { - final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of( - ALL_PRIVILEGES_GROUP, - new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.EDIT_GROUP_MEMBERS_PRIVILEGE.getType())) - )); - - return AuthorizationUtils.isAuthorized( - context.getAuthorizer(), - context.getActorUrn(), - CORP_GROUP_ENTITY_NAME, - input.getGroupUrn(), - orPrivilegeGroups); + return CompletableFuture.supplyAsync(() -> { + Origin groupOrigin = _groupService.getGroupOrigin(groupUrn); + if (groupOrigin == null || !groupOrigin.hasType()) { + try { + _groupService.migrateGroupMembershipToNativeGroupMembership(groupUrn, context.getActorUrn(), + context.getAuthentication()); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to migrate group membership when removing group members from group %s", + groupUrnStr)); + } + } else if (groupOrigin.getType() == OriginType.EXTERNAL) { + throw new RuntimeException(String.format( + "Group %s was ingested from an external provider and cannot have members manually removed from it", + groupUrnStr)); + } + try { + _groupService.removeExistingNativeGroupMembers(groupUrn, userUrnList, authentication); + return true; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupResolver.java index 5cbee1defc038e..99b75ea9d90cd4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupResolver.java @@ -5,13 +5,17 @@ import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; + /** * Resolver responsible for hard deleting a particular DataHub Corp Group */ +@Slf4j public class RemoveGroupResolver implements DataFetcher> { private final EntityClient _entityClient; @@ -28,8 +32,17 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final Urn urn = Urn.createFromString(groupUrn); return CompletableFuture.supplyAsync(() -> { try { - // TODO: Remove all dangling references to this group. _entityClient.deleteEntity(urn, context.getAuthentication()); + + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync(() -> { + try { + _entityClient.deleteEntityReferences(urn, context.getAuthentication()); + } catch (RemoteInvocationException e) { + log.error(String.format("Caught exception while attempting to clear all entity references for group with urn %s", urn), e); + } + }); + return true; } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform delete against group with urn %s", groupUrn), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java index 07b4b05f34e1e1..4ae16d850d8dbb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.graphql.generated.IngestionConfig; import com.linkedin.datahub.graphql.generated.IngestionSchedule; import com.linkedin.datahub.graphql.generated.IngestionSource; +import com.linkedin.datahub.graphql.generated.StructuredReport; import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; @@ -12,6 +13,7 @@ import com.linkedin.execution.ExecutionRequestInput; import com.linkedin.execution.ExecutionRequestResult; import com.linkedin.execution.ExecutionRequestSource; +import com.linkedin.execution.StructuredExecutionReport; import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceSchedule; @@ -19,8 +21,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class IngestionResolverUtils { public static List mapExecutionRequests(final Collection requests) { @@ -37,6 +41,7 @@ public static ExecutionRequest mapExecutionRequest(final EntityResponse entityRe final ExecutionRequest result = new ExecutionRequest(); result.setUrn(entityUrn.toString()); + result.setId(entityUrn.getId()); // Map input aspect. Must be present. final EnvelopedAspect envelopedInput = aspects.get(Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME); @@ -77,13 +82,28 @@ public static com.linkedin.datahub.graphql.generated.ExecutionRequestResult mapE result.setStartTimeMs(execRequestResult.getStartTimeMs()); result.setDurationMs(execRequestResult.getDurationMs()); result.setReport(execRequestResult.getReport()); + if (execRequestResult.hasStructuredReport()) { + result.setStructuredReport(mapStructuredReport(execRequestResult.getStructuredReport())); + } return result; } + public static StructuredReport mapStructuredReport(final StructuredExecutionReport structuredReport) { + StructuredReport structuredReportResult = new StructuredReport(); + structuredReportResult.setType(structuredReport.getType()); + structuredReportResult.setSerializedValue(structuredReport.getSerializedValue()); + structuredReportResult.setContentType(structuredReport.getContentType()); + return structuredReportResult; + } + public static List mapIngestionSources(final Collection entities) { final List results = new ArrayList<>(); for (EntityResponse response : entities) { - results.add(mapIngestionSource(response)); + try { + results.add(mapIngestionSource(response)); + } catch (IllegalStateException e) { + log.error("Unable to map ingestion source, continuing to other sources.", e); + } } return results; } @@ -95,6 +115,10 @@ public static IngestionSource mapIngestionSource(final EntityResponse ingestionS // There should ALWAYS be an info aspect. final EnvelopedAspect envelopedInfo = aspects.get(Constants.INGESTION_INFO_ASPECT_NAME); + if (envelopedInfo == null) { + throw new IllegalStateException("No ingestion source info aspect exists for urn: " + entityUrn); + } + // Bind into a strongly typed object. final DataHubIngestionSourceInfo ingestionSourceInfo = new DataHubIngestionSourceInfo(envelopedInfo.getValue().data()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java index 5716c831e31351..4c5a34cf07449f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java @@ -20,6 +20,7 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.key.ExecutionRequestKey; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; @@ -28,6 +29,8 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import org.json.JSONException; +import org.json.JSONObject; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; @@ -68,6 +71,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final UUID uuid = UUID.randomUUID(); final String uuidStr = uuid.toString(); key.setId(uuidStr); + final Urn executionRequestUrn = EntityKeyUtils.convertEntityKeyToUrn(key, Constants.EXECUTION_REQUEST_ENTITY_NAME); proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); // Fetch the original ingestion source @@ -100,7 +104,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) execInput.setRequestedAt(System.currentTimeMillis()); Map arguments = new HashMap<>(); - arguments.put(RECIPE_ARG_NAME, ingestionSourceInfo.getConfig().getRecipe()); + arguments.put(RECIPE_ARG_NAME, injectRunId(ingestionSourceInfo.getConfig().getRecipe(), executionRequestUrn.toString())); arguments.put(VERSION_ARG_NAME, ingestionSourceInfo.getConfig().hasVersion() ? ingestionSourceInfo.getConfig().getVersion() : _ingestionConfiguration.getDefaultCliVersion() @@ -123,4 +127,23 @@ public CompletableFuture get(final DataFetchingEnvironment environment) throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); }); } + + /** + * Injects an override run id into a recipe for tracking purposes. Any existing run id will be overwritten. + * + * TODO: Determine if this should be handled in the executor itself. + * + * @param runId the run id to place into the recipe + * @return a modified recipe JSON string + */ + private String injectRunId(final String originalJson, final String runId) { + try { + JSONObject obj = new JSONObject(originalJson); + obj.put("run_id", runId); + return obj.toString(); + } catch (JSONException e) { + // This should ideally never be hit. + throw new IllegalArgumentException("Failed to create execution request: Invalid recipe json provided."); + } + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java new file mode 100644 index 00000000000000..59e20fc49c0dfc --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolver.java @@ -0,0 +1,90 @@ +package com.linkedin.datahub.graphql.resolvers.ingest.execution; + +import com.linkedin.data.template.StringMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateTestConnectionRequestInput; +import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.execution.ExecutionRequestInput; +import com.linkedin.execution.ExecutionRequestSource; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.config.IngestionConfiguration; +import com.linkedin.metadata.key.ExecutionRequestKey; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/** + * Creates an on-demand ingestion execution request. + */ +public class CreateTestConnectionRequestResolver implements DataFetcher> { + + private static final String TEST_CONNECTION_TASK_NAME = "TEST_CONNECTION"; + private static final String TEST_CONNECTION_SOURCE_NAME = "MANUAL_TEST_CONNECTION"; + private static final String RECIPE_ARG_NAME = "recipe"; + private static final String VERSION_ARG_NAME = "version"; + private static final String DEFAULT_EXECUTOR_ID = "default"; + + private final EntityClient _entityClient; + private final IngestionConfiguration _ingestionConfiguration; + + public CreateTestConnectionRequestResolver(final EntityClient entityClient, final IngestionConfiguration ingestionConfiguration) { + _entityClient = entityClient; + _ingestionConfiguration = ingestionConfiguration; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + return CompletableFuture.supplyAsync(() -> { + + if (!IngestionAuthUtils.canManageIngestion(context)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + final CreateTestConnectionRequestInput input = + bindArgument(environment.getArgument("input"), CreateTestConnectionRequestInput.class); + + try { + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + final ExecutionRequestKey key = new ExecutionRequestKey(); + final UUID uuid = UUID.randomUUID(); + final String uuidStr = uuid.toString(); + key.setId(uuidStr); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + + final ExecutionRequestInput execInput = new ExecutionRequestInput(); + execInput.setTask(TEST_CONNECTION_TASK_NAME); + execInput.setSource(new ExecutionRequestSource().setType(TEST_CONNECTION_SOURCE_NAME)); + execInput.setExecutorId(DEFAULT_EXECUTOR_ID); + execInput.setRequestedAt(System.currentTimeMillis()); + + Map arguments = new HashMap<>(); + arguments.put(RECIPE_ARG_NAME, input.getRecipe()); + execInput.setArgs(new StringMap(arguments)); + + proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME); + proposal.setAspectName(Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(execInput)); + proposal.setChangeType(ChangeType.UPSERT); + + return _entityClient.ingestProposal(proposal, context.getAuthentication()); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to create new test ingestion connection request %s", input.toString()), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java index 99f0e1374f530f..8dd72a7b2182cf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DataHubSecretKey; import com.linkedin.metadata.secret.SecretService; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.secret.DataHubSecretValue; @@ -38,34 +39,36 @@ public CreateSecretResolver( @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); + final CreateSecretInput input = bindArgument(environment.getArgument("input"), CreateSecretInput.class); return CompletableFuture.supplyAsync(() -> { if (IngestionAuthUtils.canManageSecrets(context)) { - final CreateSecretInput input = bindArgument(environment.getArgument("input"), CreateSecretInput.class); + try { - final MetadataChangeProposal proposal = new MetadataChangeProposal(); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); - // Create the Ingestion source key --> use the display name as a unique id to ensure it's not duplicated. - final DataHubSecretKey key = new DataHubSecretKey(); - key.setId(input.getName()); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + // Create the Ingestion source key --> use the display name as a unique id to ensure it's not duplicated. + final DataHubSecretKey key = new DataHubSecretKey(); + key.setId(input.getName()); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); - // Create the secret value. - final DataHubSecretValue value = new DataHubSecretValue(); - value.setName(input.getName()); - value.setValue(_secretService.encrypt(input.getValue())); - value.setDescription(input.getDescription(), SetMode.IGNORE_NULL); + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.SECRETS_ENTITY_NAME), context.getAuthentication())) { + throw new IllegalArgumentException("This Secret already exists!"); + } - proposal.setEntityType(Constants.SECRETS_ENTITY_NAME); - proposal.setAspectName(Constants.SECRET_VALUE_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(value)); - proposal.setChangeType(ChangeType.UPSERT); + // Create the secret value. + final DataHubSecretValue value = new DataHubSecretValue(); + value.setName(input.getName()); + value.setValue(_secretService.encrypt(input.getValue())); + value.setDescription(input.getDescription(), SetMode.IGNORE_NULL); - System.out.println(String.format("About to ingest %s", proposal)); + proposal.setEntityType(Constants.SECRETS_ENTITY_NAME); + proposal.setAspectName(Constants.SECRET_VALUE_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(value)); + proposal.setChangeType(ChangeType.UPSERT); - try { return _entityClient.ingestProposal(proposal, context.getAuthentication()); } catch (Exception e) { throw new RuntimeException(String.format("Failed to create new secret with name %s", input.getName()), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java index 3cde649e4e88da..9e868928fe2643 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/UpsertIngestionSourceResolver.java @@ -20,6 +20,10 @@ import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import lombok.extern.slf4j.Slf4j; +import org.json.JSONException; +import org.json.JSONObject; + import java.net.URISyntaxException; import java.util.Optional; import java.util.UUID; @@ -31,6 +35,7 @@ /** * Creates or updates an ingestion source. Requires the MANAGE_INGESTION privilege. */ +@Slf4j public class UpsertIngestionSourceResolver implements DataFetcher> { private final EntityClient _entityClient; @@ -51,6 +56,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final UpdateIngestionSourceInput input = bindArgument(environment.getArgument("input"), UpdateIngestionSourceInput.class); final MetadataChangeProposal proposal = new MetadataChangeProposal(); + String ingestionSourceUrnString; if (ingestionSourceUrn.isPresent()) { // Update existing ingestion source @@ -61,6 +67,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) String.format("Malformed urn %s provided.", ingestionSourceUrn.get()), DataHubGraphQLErrorCode.BAD_REQUEST); } + ingestionSourceUrnString = ingestionSourceUrn.get(); } else { // Create new ingestion source // Since we are creating a new Ingestion Source, we need to generate a unique UUID. @@ -71,10 +78,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final DataHubIngestionSourceKey key = new DataHubIngestionSourceKey(); key.setId(uuidStr); proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + ingestionSourceUrnString = String.format("urn:li:dataHubIngestionSource:%s", uuidStr); } // Create the policy info. - final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input); + final DataHubIngestionSourceInfo info = mapIngestionSourceInfo(input, ingestionSourceUrnString); proposal.setEntityType(Constants.INGESTION_SOURCE_ENTITY_NAME); proposal.setAspectName(Constants.INGESTION_INFO_ASPECT_NAME); proposal.setAspect(GenericRecordUtils.serializeAspect(info)); @@ -90,20 +98,24 @@ public CompletableFuture get(final DataFetchingEnvironment environment) }); } - private DataHubIngestionSourceInfo mapIngestionSourceInfo(final UpdateIngestionSourceInput input) { + private DataHubIngestionSourceInfo mapIngestionSourceInfo(final UpdateIngestionSourceInput input, final String ingestionSourceUrn) { final DataHubIngestionSourceInfo result = new DataHubIngestionSourceInfo(); result.setType(input.getType()); result.setName(input.getName()); - result.setConfig(mapConfig(input.getConfig())); + result.setConfig(mapConfig(input.getConfig(), ingestionSourceUrn)); if (input.getSchedule() != null) { result.setSchedule(mapSchedule(input.getSchedule())); } return result; } - private DataHubIngestionSourceConfig mapConfig(final UpdateIngestionSourceConfigInput input) { + private DataHubIngestionSourceConfig mapConfig(final UpdateIngestionSourceConfigInput input, final String ingestionSourceUrn) { final DataHubIngestionSourceConfig result = new DataHubIngestionSourceConfig(); - result.setRecipe(input.getRecipe()); + String recipe = input.getRecipe(); + if (recipe != null) { + recipe = optionallySetPipelineName(recipe, ingestionSourceUrn); + } + result.setRecipe(recipe); if (input.getVersion() != null) { result.setVersion(input.getVersion()); } @@ -119,4 +131,19 @@ private DataHubIngestionSourceSchedule mapSchedule(final UpdateIngestionSourceSc result.setTimezone(input.getTimezone()); return result; } + + private String optionallySetPipelineName(String recipe, String ingestionSourceUrn) { + try { + JSONObject jsonRecipe = new JSONObject(recipe); + boolean hasPipelineName = jsonRecipe.has("pipeline_name") && jsonRecipe.get("pipeline_name") != null && !jsonRecipe.get("pipeline_name").equals(""); + + if (!hasPipelineName) { + jsonRecipe.put("pipeline_name", ingestionSourceUrn); + recipe = jsonRecipe.toString(); + } + } catch (JSONException e) { + log.warn("Error parsing ingestion recipe in JSON form", e); + } + return recipe; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java index 8ed18b31e66691..b6941a91125374 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java @@ -1,18 +1,20 @@ package com.linkedin.datahub.graphql.resolvers.load; -import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityLineageResult; import com.linkedin.datahub.graphql.generated.LineageDirection; import com.linkedin.datahub.graphql.generated.LineageInput; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; -import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.graph.SiblingGraphService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; @@ -21,17 +23,17 @@ * GraphQL Resolver responsible for fetching lineage relationships between entities in the DataHub graph. * Lineage relationship denotes whether an entity is directly upstream or downstream of another entity */ +@Slf4j public class EntityLineageResultResolver implements DataFetcher> { - private final GraphClient _graphClient; + private final SiblingGraphService _siblingGraphService; - public EntityLineageResultResolver(final GraphClient graphClient) { - _graphClient = graphClient; + public EntityLineageResultResolver(final SiblingGraphService siblingGraphService) { + _siblingGraphService = siblingGraphService; } @Override public CompletableFuture get(DataFetchingEnvironment environment) { - final QueryContext context = environment.getContext(); final String urn = ((Entity) environment.getSource()).getUrn(); final LineageInput input = bindArgument(environment.getArgument("input"), LineageInput.class); @@ -40,11 +42,28 @@ public CompletableFuture get(DataFetchingEnvironment enviro final Integer start = input.getStart(); // Optional! @Nullable final Integer count = input.getCount(); // Optional! + @Nullable + final Boolean separateSiblings = input.getSeparateSiblings(); // Optional! com.linkedin.metadata.graph.LineageDirection resolvedDirection = com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString()); - return CompletableFuture.supplyAsync(() -> mapEntityRelationships(lineageDirection, - _graphClient.getLineageEntities(urn, resolvedDirection, start, count, 1, context.getActorUrn()))); + + return CompletableFuture.supplyAsync(() -> { + try { + return mapEntityRelationships(lineageDirection, + _siblingGraphService.getLineage( + Urn.createFromString(urn), + resolvedDirection, + start != null ? start : 0, + count != null ? count : 100, + 1, + separateSiblings != null ? input.getSeparateSiblings() : false + )); + } catch (URISyntaxException e) { + log.error("Failed to fetch lineage for {}", urn); + throw new RuntimeException(String.format("Failed to fetch lineage for {}", urn), e); + } + }); } private EntityLineageResult mapEntityRelationships(final LineageDirection lineageDirection, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/UsageTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/UsageTypeResolver.java deleted file mode 100644 index fda80d876d88a9..00000000000000 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/UsageTypeResolver.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.linkedin.datahub.graphql.resolvers.load; - -import com.linkedin.datahub.graphql.UsageStatsKey; - -import com.linkedin.datahub.graphql.generated.Entity; -import com.linkedin.datahub.graphql.types.LoadableType; -import com.linkedin.pegasus2avro.usage.UsageQueryResult; -import com.linkedin.usage.UsageTimeRange; -import graphql.schema.DataFetcher; -import graphql.schema.DataFetchingEnvironment; -import java.util.concurrent.CompletableFuture; -import org.dataloader.DataLoader; -import lombok.extern.slf4j.Slf4j; - - -/** - * Generic GraphQL resolver responsible for - * - * 1. Retrieving a single input urn. - * 2. Resolving a single {@link LoadableType}. - * - * Note that this resolver expects that {@link DataLoader}s were registered - * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} - * - */ -@Slf4j -public class UsageTypeResolver implements DataFetcher> { - - @Override - public CompletableFuture get(DataFetchingEnvironment environment) { - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("UsageQueryResult"); - - String deprecatedResource = environment.getArgument("resource"); - if (deprecatedResource != null) { - log.info("You no longer need to provide the deprecated `resource` param to usageStats" - + "resolver. Provided: {}", deprecatedResource); - } - final String resource = ((Entity) environment.getSource()).getUrn(); - UsageTimeRange duration = UsageTimeRange.valueOf(environment.getArgument("range")); - - UsageStatsKey key = new UsageStatsKey(resource, duration); - - return loader.load(key); - } -} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java index faddf984f7bbf0..ca91cefc4d748c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnerResolver.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; @@ -7,7 +8,9 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.AddOwnerInput; import com.linkedin.datahub.graphql.generated.OwnerEntityType; +import com.linkedin.datahub.graphql.generated.OwnerInput; import com.linkedin.datahub.graphql.generated.OwnershipType; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; @@ -50,11 +53,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw log.debug("Adding Owner. input: {}", input.toString()); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - OwnerUtils.addOwner( - ownerUrn, - // Assumption Alert: Assumes that GraphQL ownership type === GMS ownership type - com.linkedin.common.OwnershipType.valueOf(type.name()), - targetUrn, + OwnerUtils.addOwnersToResources( + ImmutableList.of(new OwnerInput(input.getOwnerUrn(), ownerEntityType, type)), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), null, null)), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java index 8d9630f08ad4ed..057e6dc80ef4f4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddOwnersResolver.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; @@ -7,6 +8,7 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.AddOwnersInput; import com.linkedin.datahub.graphql.generated.OwnerInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; @@ -47,9 +49,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw log.debug("Adding Owners. input: {}", input.toString()); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - OwnerUtils.addOwners( + OwnerUtils.addOwnersToResources( owners, - targetUrn, + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), null, null)), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagResolver.java index b2420f8a446d2f..78d2341492b398 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagResolver.java @@ -6,8 +6,10 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.generated.TagAssociationInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -34,12 +36,12 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw } return CompletableFuture.supplyAsync(() -> { - LabelUtils.validateInput( + LabelUtils.validateResourceAndLabel( tagUrn, targetUrn, input.getSubResource(), input.getSubResourceType(), - "tag", + Constants.TAG_ENTITY_NAME, _entityService, false ); @@ -52,10 +54,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw log.info("Adding Tag. input: {}", input.toString()); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - LabelUtils.addTagsToTarget( + LabelUtils.addTagsToResources( ImmutableList.of(tagUrn), - targetUrn, - input.getSubResource(), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), input.getSubResourceType(), input.getSubResource())), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagsResolver.java index b8791b91732181..7174f3edffee67 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTagsResolver.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; @@ -7,7 +8,9 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.AddTagsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -40,22 +43,21 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } - LabelUtils.validateInput( + LabelUtils.validateResourceAndLabel( tagUrns, targetUrn, input.getSubResource(), input.getSubResourceType(), - "tag", + Constants.TAG_ENTITY_NAME, _entityService, false ); try { log.info("Adding Tags. input: {}", input.toString()); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - LabelUtils.addTagsToTarget( + LabelUtils.addTagsToResources( tagUrns, - targetUrn, - input.getSubResource(), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), input.getSubResourceType(), input.getSubResource())), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermResolver.java index f69a23bc5763bd..056b5db4324c34 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermResolver.java @@ -5,8 +5,10 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.generated.TermAssociationInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -32,12 +34,12 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw } return CompletableFuture.supplyAsync(() -> { - LabelUtils.validateInput( + LabelUtils.validateResourceAndLabel( termUrn, targetUrn, input.getSubResource(), input.getSubResourceType(), - "glossaryTerm", + Constants.GLOSSARY_TERM_ENTITY_NAME, _entityService, false ); @@ -45,10 +47,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { log.info("Adding Term. input: {}", input); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - LabelUtils.addTermsToTarget( + LabelUtils.addTermsToResources( ImmutableList.of(termUrn), - targetUrn, - input.getSubResource(), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), input.getSubResourceType(), input.getSubResource())), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermsResolver.java index 3676c589c3c648..2f58b6b09e681b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/AddTermsResolver.java @@ -1,12 +1,15 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.AddTermsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -37,12 +40,12 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } - LabelUtils.validateInput( + LabelUtils.validateResourceAndLabel( termUrns, targetUrn, input.getSubResource(), input.getSubResourceType(), - "glossaryTerm", + Constants.GLOSSARY_TERM_ENTITY_NAME, _entityService, false ); @@ -50,10 +53,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw try { log.info("Adding Term. input: {}", input); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - LabelUtils.addTermsToTarget( + LabelUtils.addTermsToResources( termUrns, - targetUrn, - input.getSubResource(), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), input.getSubResourceType(), input.getSubResource())), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java new file mode 100644 index 00000000000000..ae1ba8a50ab011 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddOwnersResolver.java @@ -0,0 +1,90 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchAddOwnersInput; +import com.linkedin.datahub.graphql.generated.OwnerInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchAddOwnersResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final BatchAddOwnersInput input = bindArgument(environment.getArgument("input"), BatchAddOwnersInput.class); + final List owners = input.getOwners(); + final List resources = input.getResources(); + final QueryContext context = environment.getContext(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the batch + validateOwners(owners); + validateInputResources(resources, context); + + try { + // Then execute the bulk add + batchAddOwners(owners, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateOwners(List owners) { + for (OwnerInput ownerInput : owners) { + OwnerUtils.validateOwner(UrnUtils.getUrn(ownerInput.getOwnerUrn()), ownerInput.getOwnerEntityType(), _entityService); + } + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + + if (resource.getSubResource() != null) { + throw new IllegalArgumentException("Malformed input provided: owners cannot be applied to subresources."); + } + + if (!OwnerUtils.isAuthorizedToUpdateOwners(context, resourceUrn)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchAddOwners(List owners, List resources, QueryContext context) { + log.debug("Batch adding owners. owners: {}, resources: {}", owners, resources); + try { + OwnerUtils.addOwnersToResources(owners, resources, UrnUtils.getUrn(context.getActorUrn()), _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to batch add Owners %s to resources with urns %s!", + owners, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddTagsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddTagsResolver.java new file mode 100644 index 00000000000000..a7d101bf128f69 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddTagsResolver.java @@ -0,0 +1,86 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchAddTagsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchAddTagsResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchAddTagsInput input = bindArgument(environment.getArgument("input"), BatchAddTagsInput.class); + final List tagUrns = input.getTagUrns().stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + final List resources = input.getResources(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the batch + validateTags(tagUrns); + validateInputResources(resources, context); + + try { + // Then execute the bulk add + batchAddTags(tagUrns, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateTags(List tagUrns) { + for (Urn tagUrn : tagUrns) { + LabelUtils.validateLabel(tagUrn, Constants.TAG_ENTITY_NAME, _entityService); + } + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + if (!LabelUtils.isAuthorizedToUpdateTags(context, resourceUrn, resource.getSubResource())) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchAddTags(List tagUrns, List resources, QueryContext context) { + log.debug("Batch adding Tags. tags: {}, resources: {}", resources, tagUrns); + try { + LabelUtils.addTagsToResources(tagUrns, resources, UrnUtils.getUrn(context.getActorUrn()), _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to batch add Tags %s to resources with urns %s!", + tagUrns, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddTermsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddTermsResolver.java new file mode 100644 index 00000000000000..3f01045a8bbd23 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchAddTermsResolver.java @@ -0,0 +1,86 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchAddTermsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchAddTermsResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchAddTermsInput input = bindArgument(environment.getArgument("input"), BatchAddTermsInput.class); + final List termUrns = input.getTermUrns().stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + final List resources = input.getResources(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the batch + validateTerms(termUrns); + validateInputResources(resources, context); + + try { + // Then execute the bulk add + batchAddTerms(termUrns, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateTerms(List termUrns) { + for (Urn termUrn : termUrns) { + LabelUtils.validateLabel(termUrn, Constants.GLOSSARY_TERM_ENTITY_NAME, _entityService); + } + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + if (!LabelUtils.isAuthorizedToUpdateTerms(context, resourceUrn, resource.getSubResource())) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchAddTerms(List termUrns, List resources, QueryContext context) { + log.debug("Batch adding Terms. terms: {}, resources: {}", resources, termUrns); + try { + LabelUtils.addTermsToResources(termUrns, resources, UrnUtils.getUrn(context.getActorUrn()), _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to batch add Terms %s to resources with urns %s!", + termUrns, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveOwnersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveOwnersResolver.java new file mode 100644 index 00000000000000..53ac37800848cc --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveOwnersResolver.java @@ -0,0 +1,83 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchRemoveOwnersInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchRemoveOwnersResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final BatchRemoveOwnersInput input = bindArgument(environment.getArgument("input"), BatchRemoveOwnersInput.class); + final List owners = input.getOwnerUrns(); + final List resources = input.getResources(); + final QueryContext context = environment.getContext(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the batch + validateInputResources(resources, context); + + try { + // Then execute the bulk remove + batchRemoveOwners(owners, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + + if (resource.getSubResource() != null) { + throw new IllegalArgumentException("Malformed input provided: owners cannot be removed from subresources."); + } + + if (!OwnerUtils.isAuthorizedToUpdateOwners(context, resourceUrn)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchRemoveOwners(List ownerUrns, List resources, QueryContext context) { + log.debug("Batch removing owners. owners: {}, resources: {}", ownerUrns, resources); + try { + OwnerUtils.removeOwnersFromResources(ownerUrns.stream().map(UrnUtils::getUrn).collect( + Collectors.toList()), resources, UrnUtils.getUrn(context.getActorUrn()), _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to batch remove Owners %s to resources with urns %s!", + ownerUrns, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveTagsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveTagsResolver.java new file mode 100644 index 00000000000000..ab432f0afcaec0 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveTagsResolver.java @@ -0,0 +1,78 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchRemoveTagsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchRemoveTagsResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchRemoveTagsInput input = bindArgument(environment.getArgument("input"), BatchRemoveTagsInput.class); + final List tagUrns = input.getTagUrns().stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + final List resources = input.getResources(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the batch + validateInputResources(resources, context); + + try { + // Then execute the bulk add + batchRemoveTags(tagUrns, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + if (!LabelUtils.isAuthorizedToUpdateTags(context, resourceUrn, resource.getSubResource())) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchRemoveTags(List tagUrns, List resources, QueryContext context) { + log.debug("Batch removing Tags. tags: {}, resources: {}", resources, tagUrns); + try { + LabelUtils.removeTagsFromResources(tagUrns, resources, UrnUtils.getUrn(context.getActorUrn()), _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to remove Tags %s to resources with urns %s!", + tagUrns, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveTermsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveTermsResolver.java new file mode 100644 index 00000000000000..c8870cc44bf9e6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchRemoveTermsResolver.java @@ -0,0 +1,78 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchRemoveTermsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchRemoveTermsResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchRemoveTermsInput input = bindArgument(environment.getArgument("input"), BatchRemoveTermsInput.class); + final List termUrns = input.getTermUrns().stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList()); + final List resources = input.getResources(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the batch + validateInputResources(resources, context); + + try { + // Then execute the bulk add + batchRemoveTerms(termUrns, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + if (!LabelUtils.isAuthorizedToUpdateTerms(context, resourceUrn, resource.getSubResource())) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchRemoveTerms(List termUrns, List resources, QueryContext context) { + log.debug("Batch removing Terms. terms: {}, resources: {}", resources, termUrns); + try { + LabelUtils.removeTermsFromResources(termUrns, resources, UrnUtils.getUrn(context.getActorUrn()), _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to remove Terms %s to resources with urns %s!", + termUrns, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchSetDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchSetDomainResolver.java new file mode 100644 index 00000000000000..9b6167c673d8db --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchSetDomainResolver.java @@ -0,0 +1,88 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchSetDomainInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchSetDomainResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchSetDomainInput input = bindArgument(environment.getArgument("input"), BatchSetDomainInput.class); + final String maybeDomainUrn = input.getDomainUrn(); + final List resources = input.getResources(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the domain + validateDomain(maybeDomainUrn); + validateInputResources(resources, context); + + try { + // Then execute the bulk add + batchSetDomains(maybeDomainUrn, resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateDomain(@Nullable String maybeDomainUrn) { + if (maybeDomainUrn != null) { + DomainUtils.validateDomain(UrnUtils.getUrn(maybeDomainUrn), _entityService); + } + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + if (!DomainUtils.isAuthorizedToUpdateDomainsForEntity(context, resourceUrn)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchSetDomains(String maybeDomainUrn, List resources, QueryContext context) { + log.debug("Batch adding Domains. domainUrn: {}, resources: {}", maybeDomainUrn, resources); + try { + DomainUtils.setDomainForResources(maybeDomainUrn == null ? null : UrnUtils.getUrn(maybeDomainUrn), + resources, + UrnUtils.getUrn(context.getActorUrn()), + _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to batch set Domain %s to resources with urns %s!", + maybeDomainUrn, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchUpdateDeprecationResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchUpdateDeprecationResolver.java new file mode 100644 index 00000000000000..5961dc9087a638 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchUpdateDeprecationResolver.java @@ -0,0 +1,87 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchUpdateDeprecationInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DeprecationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchUpdateDeprecationResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchUpdateDeprecationInput input = bindArgument(environment.getArgument("input"), BatchUpdateDeprecationInput.class); + final List resources = input.getResources(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the resources + validateInputResources(resources, context); + + try { + // Then execute the bulk update + batchUpdateDeprecation(input.getDeprecated(), input.getNote(), input.getDecommissionTime(), resources, context); + return true; + } catch (Exception e) { + log.error("Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } + + private void validateInputResources(List resources, QueryContext context) { + for (ResourceRefInput resource : resources) { + validateInputResource(resource, context); + } + } + + private void validateInputResource(ResourceRefInput resource, QueryContext context) { + final Urn resourceUrn = UrnUtils.getUrn(resource.getResourceUrn()); + if (!DeprecationUtils.isAuthorizedToUpdateDeprecationForEntity(context, resourceUrn)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + LabelUtils.validateResource(resourceUrn, resource.getSubResource(), resource.getSubResourceType(), _entityService); + } + + private void batchUpdateDeprecation(boolean deprecated, + @Nullable String note, + @Nullable Long decommissionTime, + List resources, + QueryContext context) { + log.debug("Batch updating deprecation. deprecated: {}, note: {}, decommissionTime: {}, resources: {}", deprecated, note, decommissionTime, resources); + try { + DeprecationUtils.updateDeprecationForResources( + deprecated, + note, + decommissionTime, + resources, + UrnUtils.getUrn(context.getActorUrn()), + _entityService); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to batch update deprecated to %s for resources with urns %s!", + deprecated, + resources.stream().map(ResourceRefInput::getResourceUrn).collect(Collectors.toList())), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchUpdateSoftDeletedResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchUpdateSoftDeletedResolver.java new file mode 100644 index 00000000000000..69b2b92fb9ccaf --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/BatchUpdateSoftDeletedResolver.java @@ -0,0 +1,79 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.BatchUpdateSoftDeletedInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DeleteUtils; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@RequiredArgsConstructor +public class BatchUpdateSoftDeletedResolver implements DataFetcher> { + + private final EntityService _entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final BatchUpdateSoftDeletedInput input = bindArgument(environment.getArgument("input"), BatchUpdateSoftDeletedInput.class); + final List urns = input.getUrns(); + final boolean deleted = input.getDeleted(); + + return CompletableFuture.supplyAsync(() -> { + + // First, validate the entities exist + validateInputUrns(urns, context); + + try { + // Then execute the bulk soft delete + batchUpdateSoftDeleted(deleted, urns, context); + return true; + } catch (Exception e) { + log.error("Failed to perform batch soft delete against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform batch soft delete against input %s", input.toString()), e); + } + }); + } + + private void validateInputUrns(List urnStrs, QueryContext context) { + for (String urnStr : urnStrs) { + validateInputUrn(urnStr, context); + } + } + + private void validateInputUrn(String urnStr, QueryContext context) { + final Urn urn = UrnUtils.getUrn(urnStr); + if (!DeleteUtils.isAuthorizedToDeleteEntity(context, urn)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + if (!_entityService.exists(urn)) { + throw new IllegalArgumentException(String.format("Failed to soft delete entity with urn %s. Entity does not exist.", urn)); + } + } + + private void batchUpdateSoftDeleted(boolean removed, List urnStrs, QueryContext context) { + log.debug("Batch soft deleting assets. urns: {}", urnStrs); + try { + DeleteUtils.updateStatusForResources( + removed, + urnStrs, + UrnUtils.getUrn(context.getActorUrn()), + _entityService); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to batch update soft deleted status entities with urns %s!", urnStrs), + e); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolver.java new file mode 100644 index 00000000000000..30bd940a7dfed4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolver.java @@ -0,0 +1,53 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.codahale.metrics.Timer; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.types.BatchMutableType; +import com.linkedin.metadata.utils.metrics.MetricUtils; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/** + * Generic GraphQL resolver responsible for performing updates against particular types. + * + * @param the generated GraphQL POJO corresponding to the input type. + * @param the generated GraphQL POJO corresponding to the return type. + */ +public class MutableTypeBatchResolver implements DataFetcher>> { + + private static final Logger _logger = LoggerFactory.getLogger(MutableTypeBatchResolver.class.getName()); + + private final BatchMutableType _batchMutableType; + + public MutableTypeBatchResolver(final BatchMutableType batchMutableType) { + _batchMutableType = batchMutableType; + } + + @Override + public CompletableFuture> get(DataFetchingEnvironment environment) throws Exception { + final B[] input = bindArgument(environment.getArgument("input"), _batchMutableType.batchInputClass()); + + return CompletableFuture.supplyAsync(() -> { + Timer.Context timer = MetricUtils.timer(this.getClass(), "batchMutate").time(); + + try { + return _batchMutableType.batchUpdate(input, environment.getContext()); + } catch (AuthorizationException e) { + throw e; + } catch (Exception e) { + _logger.error("Failed to perform batchUpdate", e); + throw new IllegalArgumentException(e); + } finally { + timer.stop(); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java index 39bc08cf21ce15..ab7f645887f903 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MutationUtils.java @@ -33,6 +33,16 @@ public static void persistAspect(Urn urn, String aspectName, RecordTemplate aspe entityService.ingestProposal(proposal, getAuditStamp(actor)); } + public static MetadataChangeProposal buildMetadataChangeProposal(Urn urn, String aspectName, RecordTemplate aspect, Urn actor, EntityService entityService) { + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(urn); + proposal.setEntityType(urn.getEntityType()); + proposal.setAspectName(aspectName); + proposal.setAspect(GenericRecordUtils.serializeAspect(aspect)); + proposal.setChangeType(ChangeType.UPSERT); + return proposal; + } + public static RecordTemplate getAspectFromEntity(String entityUrn, String aspectName, EntityService entityService, RecordTemplate defaultValue) { try { RecordTemplate aspect = entityService.getAspect( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveOwnerResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveOwnerResolver.java index 1aa16e094151cb..4df5d27ebe8bcc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveOwnerResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveOwnerResolver.java @@ -1,10 +1,12 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.RemoveOwnerInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; @@ -39,12 +41,10 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw _entityService ); try { - log.debug("Removing Link input: {}", input); - Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - OwnerUtils.removeOwner( - ownerUrn, - targetUrn, + OwnerUtils.removeOwnersFromResources( + ImmutableList.of(ownerUrn), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), null, null)), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTagResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTagResolver.java index c58f113950809a..33a95c35760614 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTagResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTagResolver.java @@ -1,11 +1,14 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.generated.TagAssociationInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -32,28 +35,27 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw } return CompletableFuture.supplyAsync(() -> { - LabelUtils.validateInput( + LabelUtils.validateResourceAndLabel( tagUrn, targetUrn, input.getSubResource(), input.getSubResourceType(), - "tag", + Constants.TAG_ENTITY_NAME, _entityService, true ); try { - if (!tagUrn.getEntityType().equals("tag")) { + if (!tagUrn.getEntityType().equals(Constants.TAG_ENTITY_NAME)) { log.error("Failed to remove %s. It is not a tag urn.", tagUrn.toString()); return false; } - log.info("Removing Tag. input: %s", input); + log.debug("Removing Tag. input: %s", input); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - LabelUtils.removeTagFromTarget( - tagUrn, - targetUrn, - input.getSubResource(), + LabelUtils.removeTagsFromResources( + ImmutableList.of(tagUrn), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), input.getSubResourceType(), input.getSubResource())), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTermResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTermResolver.java index 92174707a4464b..8f18b0ecd61989 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTermResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/RemoveTermResolver.java @@ -1,11 +1,14 @@ package com.linkedin.datahub.graphql.resolvers.mutate; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.generated.TermAssociationInput; import com.linkedin.datahub.graphql.resolvers.mutate.util.LabelUtils; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -32,12 +35,12 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw } return CompletableFuture.supplyAsync(() -> { - LabelUtils.validateInput( + LabelUtils.validateResourceAndLabel( termUrn, targetUrn, input.getSubResource(), input.getSubResourceType(), - "glossaryTerm", + Constants.GLOSSARY_TERM_ENTITY_NAME, _entityService, true ); @@ -51,10 +54,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw log.info(String.format("Removing Term. input: {}", input)); Urn actor = CorpuserUrn.createFromString(((QueryContext) environment.getContext()).getActorUrn()); - LabelUtils.removeTermFromTarget( - termUrn, - targetUrn, - input.getSubResource(), + LabelUtils.removeTermsFromResources( + ImmutableList.of(termUrn), + ImmutableList.of(new ResourceRefInput(input.getResourceUrn(), input.getSubResourceType(), input.getSubResource())), actor, _entityService ); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java index 93e2c9d9e6b75b..6529a3a66bfa84 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java @@ -7,6 +7,7 @@ import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.UpdateNameInput; +import com.linkedin.domain.DomainProperties; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.metadata.Constants; @@ -44,6 +45,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw return updateGlossaryTermName(targetUrn, input, environment.getContext()); case Constants.GLOSSARY_NODE_ENTITY_NAME: return updateGlossaryNodeName(targetUrn, input, environment.getContext()); + case Constants.DOMAIN_ENTITY_NAME: + return updateDomainName(targetUrn, input, environment.getContext()); default: throw new RuntimeException( String.format("Failed to update name. Unsupported resource type %s provided.", targetUrn)); @@ -98,4 +101,28 @@ private Boolean updateGlossaryNodeName( } throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } + + private Boolean updateDomainName( + Urn targetUrn, + UpdateNameInput input, + QueryContext context + ) { + if (AuthorizationUtils.canManageDomains(context)) { + try { + DomainProperties domainProperties = (DomainProperties) getAspectFromEntity( + targetUrn.toString(), Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, null); + if (domainProperties == null) { + throw new IllegalArgumentException("Domain does not exist"); + } + domainProperties.setName(input.getName()); + Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); + persistAspect(targetUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, domainProperties, actor, _entityService); + + return true; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to perform update against input %s", input), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java new file mode 100644 index 00000000000000..75c54277e2b9d7 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeleteUtils.java @@ -0,0 +1,80 @@ +package com.linkedin.datahub.graphql.resolvers.mutate.util; + +import com.google.common.collect.ImmutableList; + +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + + +@Slf4j +public class DeleteUtils { + private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() + )); + + private DeleteUtils() { } + + public static boolean isAuthorizedToDeleteEntity(@Nonnull QueryContext context, Urn entityUrn) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of( + ALL_PRIVILEGES_GROUP, + new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE.getType())) + )); + + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + entityUrn.getEntityType(), + entityUrn.toString(), + orPrivilegeGroups); + } + + public static void updateStatusForResources( + boolean removed, + List urnStrs, + Urn actor, + EntityService entityService + ) { + final List changes = new ArrayList<>(); + for (String urnStr : urnStrs) { + changes.add(buildSoftDeleteProposal(removed, urnStr, actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); + } + + private static MetadataChangeProposal buildSoftDeleteProposal( + boolean removed, + String urnStr, + Urn actor, + EntityService entityService + ) { + Status status = (Status) getAspectFromEntity( + urnStr, + Constants.STATUS_ASPECT_NAME, + entityService, + new Status()); + status.setRemoved(removed); + return buildMetadataChangeProposal(UrnUtils.getUrn(urnStr), Constants.STATUS_ASPECT_NAME, status, actor, entityService); + } + + private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { + // TODO: Replace this with a batch ingest proposals endpoint. + for (MetadataChangeProposal change : changes) { + entityService.ingestProposal(change, getAuditStamp(actor)); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java new file mode 100644 index 00000000000000..48af0b401084e4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DeprecationUtils.java @@ -0,0 +1,95 @@ +package com.linkedin.datahub.graphql.resolvers.mutate.util; + +import com.google.common.collect.ImmutableList; + +import com.linkedin.common.Deprecation; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + + +@Slf4j +public class DeprecationUtils { + private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( + PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() + )); + + private DeprecationUtils() { } + + public static boolean isAuthorizedToUpdateDeprecationForEntity(@Nonnull QueryContext context, Urn entityUrn) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = new DisjunctivePrivilegeGroup(ImmutableList.of( + ALL_PRIVILEGES_GROUP, + new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.EDIT_ENTITY_DEPRECATION_PRIVILEGE.getType())) + )); + + return AuthorizationUtils.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + entityUrn.getEntityType(), + entityUrn.toString(), + orPrivilegeGroups); + } + + public static void updateDeprecationForResources( + boolean deprecated, + @Nullable String note, + @Nullable Long decommissionTime, + List resources, + Urn actor, + EntityService entityService + ) { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildUpdateDeprecationProposal(deprecated, note, decommissionTime, resource, actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); + } + + private static MetadataChangeProposal buildUpdateDeprecationProposal( + boolean deprecated, + @Nullable String note, + @Nullable Long decommissionTime, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) { + Deprecation deprecation = (Deprecation) getAspectFromEntity( + resource.getResourceUrn(), + Constants.DEPRECATION_ASPECT_NAME, + entityService, + new Deprecation()); + deprecation.setActor(actor); + deprecation.setDeprecated(deprecated); + deprecation.setDecommissionTime(decommissionTime, SetMode.REMOVE_IF_NULL); + if (note != null) { + deprecation.setNote(note); + } else { + // Note is required field in GMS. Set to empty string if not provided. + deprecation.setNote(""); + } + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), Constants.DEPRECATION_ASPECT_NAME, deprecation, actor, entityService); + } + + private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { + // TODO: Replace this with a batch ingest proposals endpoint. + for (MetadataChangeProposal change : changes) { + entityService.ingestProposal(change, getAuditStamp(actor)); + } + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index a79517a4f049c2..89d615bd8488c2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -2,15 +2,27 @@ import com.google.common.collect.ImmutableList; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.domain.Domains; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; + @Slf4j public class DomainUtils { @@ -33,4 +45,49 @@ public static boolean isAuthorizedToUpdateDomainsForEntity(@Nonnull QueryContext entityUrn.toString(), orPrivilegeGroups); } + + public static void setDomainForResources( + @Nullable Urn domainUrn, + List resources, + Urn actor, + EntityService entityService + ) throws Exception { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildSetDomainProposal(domainUrn, resource, actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); + } + + private static MetadataChangeProposal buildSetDomainProposal( + @Nullable Urn domainUrn, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) { + Domains domains = (Domains) getAspectFromEntity( + resource.getResourceUrn(), + Constants.DOMAINS_ASPECT_NAME, + entityService, + new Domains()); + final UrnArray newDomains = new UrnArray(); + if (domainUrn != null) { + newDomains.add(domainUrn); + } + domains.setDomains(newDomains); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), Constants.DOMAINS_ASPECT_NAME, domains, actor, entityService); + } + + public static void validateDomain(Urn domainUrn, EntityService entityService) { + if (!entityService.exists(domainUrn)) { + throw new IllegalArgumentException(String.format("Failed to validate Domain with urn %s. Urn does not exist.", domainUrn)); + } + } + + private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { + // TODO: Replace this with a batch ingest proposals endpoint. + for (MetadataChangeProposal change : changes) { + entityService.ingestProposal(change, getAuditStamp(actor)); + } + } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java index 0213bddece60f8..a19a9c6e9a9c09 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java @@ -1,8 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.mutate.util; import com.google.common.collect.ImmutableList; - - import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.GlossaryTermAssociationArray; @@ -12,14 +10,17 @@ import com.linkedin.common.urn.GlossaryTermUrn; import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.generated.SubResourceType; import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.schema.EditableSchemaFieldInfo; import com.linkedin.schema.EditableSchemaMetadata; import java.net.URISyntaxException; @@ -43,9 +44,9 @@ private LabelUtils() { } public static final String EDITABLE_SCHEMA_METADATA = "editableSchemaMetadata"; public static final String TAGS_ASPECT_NAME = "globalTags"; - public static void removeTermFromTarget( + public static void removeTermFromResource( Urn labelUrn, - Urn targetUrn, + Urn resourceUrn, String subResource, Urn actor, EntityService entityService @@ -53,109 +54,99 @@ public static void removeTermFromTarget( if (subResource == null || subResource.equals("")) { com.linkedin.common.GlossaryTerms terms = (com.linkedin.common.GlossaryTerms) MutationUtils.getAspectFromEntity( - targetUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); + resourceUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); terms.setAuditStamp(getAuditStamp(actor)); removeTermIfExists(terms, labelUrn); - persistAspect(targetUrn, GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); + persistAspect(resourceUrn, GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); } else { com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( - targetUrn.toString(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); + resourceUrn.toString(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, subResource); if (!editableFieldInfo.hasGlossaryTerms()) { editableFieldInfo.setGlossaryTerms(new GlossaryTerms()); } removeTermIfExists(editableFieldInfo.getGlossaryTerms(), labelUrn); - persistAspect(targetUrn, EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + persistAspect(resourceUrn, EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); } } - public static void removeTagFromTarget( - Urn labelUrn, - Urn targetUrn, - String subResource, + public static void removeTagsFromResources( + List tags, + List resources, Urn actor, EntityService entityService - ) { - if (subResource == null || subResource.equals("")) { - com.linkedin.common.GlobalTags tags = - (com.linkedin.common.GlobalTags) getAspectFromEntity(targetUrn.toString(), TAGS_ASPECT_NAME, entityService, new GlobalTags()); - - if (!tags.hasTags()) { - tags.setTags(new TagAssociationArray()); - } - removeTagIfExists(tags, labelUrn); - persistAspect(targetUrn, TAGS_ASPECT_NAME, tags, actor, entityService); - } else { - com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = - (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( - targetUrn.toString(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); - EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, subResource); - - if (!editableFieldInfo.hasGlobalTags()) { - editableFieldInfo.setGlobalTags(new GlobalTags()); - } - removeTagIfExists(editableFieldInfo.getGlobalTags(), labelUrn); - persistAspect(targetUrn, EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + ) throws Exception { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildRemoveTagsProposal(tags, resource, actor, entityService)); } + ingestChangeProposals(changes, entityService, actor); } - public static void addTagsToTarget( - List labelUrns, - Urn targetUrn, - String subResource, + public static void addTagsToResources( + List tagUrns, + List resources, Urn actor, EntityService entityService - ) throws URISyntaxException { - if (subResource == null || subResource.equals("")) { - com.linkedin.common.GlobalTags tags = - (com.linkedin.common.GlobalTags) getAspectFromEntity(targetUrn.toString(), TAGS_ASPECT_NAME, entityService, new GlobalTags()); - - if (!tags.hasTags()) { - tags.setTags(new TagAssociationArray()); - } - addTagsIfNotExists(tags, labelUrns); - persistAspect(targetUrn, TAGS_ASPECT_NAME, tags, actor, entityService); - } else { - com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = - (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( - targetUrn.toString(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); - EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, subResource); + ) throws Exception { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildAddTagsProposal(tagUrns, resource, actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); + } - if (!editableFieldInfo.hasGlobalTags()) { - editableFieldInfo.setGlobalTags(new GlobalTags()); - } + public static void removeTermsFromResources( + List termUrns, + List resources, + Urn actor, + EntityService entityService + ) throws Exception { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildRemoveTermsProposal(termUrns, resource, actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); + } - addTagsIfNotExists(editableFieldInfo.getGlobalTags(), labelUrns); - persistAspect(targetUrn, EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + public static void addTermsToResources( + List termUrns, + List resources, + Urn actor, + EntityService entityService + ) throws Exception { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildAddTermsProposal(termUrns, resource, actor, entityService)); } + ingestChangeProposals(changes, entityService, actor); } - public static void addTermsToTarget( + public static void addTermsToResource( List labelUrns, - Urn targetUrn, + Urn resourceUrn, String subResource, Urn actor, EntityService entityService ) throws URISyntaxException { if (subResource == null || subResource.equals("")) { com.linkedin.common.GlossaryTerms terms = - (com.linkedin.common.GlossaryTerms) getAspectFromEntity(targetUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); + (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resourceUrn.toString(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); terms.setAuditStamp(getAuditStamp(actor)); if (!terms.hasTerms()) { terms.setTerms(new GlossaryTermAssociationArray()); } - addTermsIfNotExistsToEntity(terms, labelUrns); - System.out.println(String.format("Persisting terms! %s", terms.toString())); - persistAspect(targetUrn, GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); + addTermsIfNotExists(terms, labelUrns); + persistAspect(resourceUrn, GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); } else { com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( - targetUrn.toString(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); + resourceUrn.toString(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, subResource); if (!editableFieldInfo.hasGlossaryTerms()) { @@ -164,47 +155,19 @@ public static void addTermsToTarget( editableFieldInfo.getGlossaryTerms().setAuditStamp(getAuditStamp(actor)); - addTermsIfNotExistsToEntity(editableFieldInfo.getGlossaryTerms(), labelUrns); - persistAspect(targetUrn, EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + addTermsIfNotExists(editableFieldInfo.getGlossaryTerms(), labelUrns); + persistAspect(resourceUrn, EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); } } - private static void addTermsIfNotExistsToEntity(GlossaryTerms terms, List termUrns) - throws URISyntaxException { - if (!terms.hasTerms()) { - terms.setTerms(new GlossaryTermAssociationArray()); - } - - GlossaryTermAssociationArray termArray = terms.getTerms(); - - List termsToAdd = new ArrayList<>(); - for (Urn termUrn : termUrns) { - if (termArray.stream().anyMatch(association -> association.getUrn().equals(termUrn))) { - continue; - } - termsToAdd.add(termUrn); - } - - // Check for no terms to add - if (termsToAdd.size() == 0) { - return; - } - - for (Urn termUrn : termsToAdd) { - GlossaryTermAssociation newAssociation = new GlossaryTermAssociation(); - newAssociation.setUrn(GlossaryTermUrn.createFromUrn(termUrn)); - termArray.add(newAssociation); - } - } - - private static TagAssociationArray removeTagIfExists(GlobalTags tags, Urn tagUrn) { + private static TagAssociationArray removeTagsIfExists(GlobalTags tags, List tagUrns) { if (!tags.hasTags()) { tags.setTags(new TagAssociationArray()); } - TagAssociationArray tagAssociationArray = tags.getTags(); - - tagAssociationArray.removeIf(association -> association.getTag().equals(tagUrn)); + for (Urn tagUrn : tagUrns) { + tagAssociationArray.removeIf(association -> association.getTag().equals(tagUrn)); + } return tagAssociationArray; } @@ -219,33 +182,6 @@ private static GlossaryTermAssociationArray removeTermIfExists(GlossaryTerms ter return termArray; } - private static void addTagsIfNotExists(GlobalTags tags, List tagUrns) throws URISyntaxException { - if (!tags.hasTags()) { - tags.setTags(new TagAssociationArray()); - } - - TagAssociationArray tagAssociationArray = tags.getTags(); - - List tagsToAdd = new ArrayList<>(); - for (Urn tagUrn : tagUrns) { - if (tagAssociationArray.stream().anyMatch(association -> association.getTag().equals(tagUrn))) { - continue; - } - tagsToAdd.add(tagUrn); - } - - // Check for no tags to add - if (tagsToAdd.size() == 0) { - return; - } - - for (Urn tagUrn : tagsToAdd) { - TagAssociation newAssociation = new TagAssociation(); - newAssociation.setTag(TagUrn.createFromUrn(tagUrn)); - tagAssociationArray.add(newAssociation); - } - } - public static boolean isAuthorizedToUpdateTags(@Nonnull QueryContext context, Urn targetUrn, String subResource) { Boolean isTargetingSchema = subResource != null && subResource.length() > 0; @@ -288,9 +224,9 @@ public static boolean isAuthorizedToUpdateTerms(@Nonnull QueryContext context, U orPrivilegeGroups); } - public static Boolean validateInput( + public static void validateResourceAndLabel( List labelUrns, - Urn targetUrn, + Urn resourceUrn, String subResource, SubResourceType subResourceType, String labelEntityType, @@ -298,48 +234,328 @@ public static Boolean validateInput( Boolean isRemoving ) { for (Urn urn : labelUrns) { - boolean labelResult = validateInput(urn, targetUrn, subResource, subResourceType, labelEntityType, entityService, isRemoving); - if (!labelResult) { - return false; + validateResourceAndLabel(urn, resourceUrn, subResource, subResourceType, labelEntityType, entityService, isRemoving); + } + } + + public static void validateLabel(Urn labelUrn, String labelEntityType, EntityService entityService) { + if (!labelUrn.getEntityType().equals(labelEntityType)) { + throw new IllegalArgumentException(String.format("Failed to validate label with urn %s. Urn type does not match entity type %s..", + labelUrn, + labelEntityType)); + } + if (!entityService.exists(labelUrn)) { + throw new IllegalArgumentException(String.format("Failed to validate label with urn %s. Urn does not exist.", labelUrn)); + } + } + + // TODO: Move this out into a separate utilities class. + public static void validateResource(Urn resourceUrn, String subResource, SubResourceType subResourceType, EntityService entityService) { + if (!entityService.exists(resourceUrn)) { + throw new IllegalArgumentException(String.format("Failed to update resource with urn %s. Entity does not exist.", resourceUrn)); + } + if ((subResource != null && subResource.length() > 0) || subResourceType != null) { + if (subResource == null || subResource.length() == 0) { + throw new IllegalArgumentException(String.format( + "Failed to update resource with urn %s. SubResourceType (%s) provided without a subResource.", resourceUrn, subResourceType)); + } + if (subResourceType == null) { + throw new IllegalArgumentException(String.format( + "Failed to updates resource with urn %s. SubResource (%s) provided without a subResourceType.", resourceUrn, subResource)); } + validateSubresourceExists(resourceUrn, subResource, subResourceType, entityService); } - return true; } - public static Boolean validateInput( + public static void validateResourceAndLabel( Urn labelUrn, - Urn targetUrn, + Urn resourceUrn, String subResource, SubResourceType subResourceType, String labelEntityType, EntityService entityService, Boolean isRemoving ) { - if (!labelUrn.getEntityType().equals(labelEntityType)) { - throw new IllegalArgumentException(String.format("Failed to update %s on %s. Was expecting a %s.", labelUrn, targetUrn, labelEntityType)); + if (!isRemoving) { + validateLabel(labelUrn, labelEntityType, entityService); + } + validateResource(resourceUrn, subResource, subResourceType, entityService); + } + + private static MetadataChangeProposal buildAddTagsProposal( + List tagUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + if (resource.getSubResource() == null || resource.getSubResource().equals("")) { + // Case 1: Adding tags to a top-level entity + return buildAddTagsToEntityProposal(tagUrns, resource, actor, entityService); + } else { + // Case 2: Adding tags to subresource (e.g. schema fields) + return buildAddTagsToSubResourceProposal(tagUrns, resource, actor, entityService); } + } - if (!entityService.exists(targetUrn)) { - throw new IllegalArgumentException(String.format("Failed to update %s on %s. %s does not exist.", labelUrn, targetUrn, targetUrn)); + private static MetadataChangeProposal buildRemoveTagsProposal( + List tagUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + if (resource.getSubResource() == null || resource.getSubResource().equals("")) { + // Case 1: Adding tags to a top-level entity + return buildRemoveTagsToEntityProposal(tagUrns, resource, actor, entityService); + } else { + // Case 2: Adding tags to subresource (e.g. schema fields) + return buildRemoveTagsToSubResourceProposal(tagUrns, resource, actor, entityService); } + } + + private static MetadataChangeProposal buildRemoveTagsToEntityProposal( + List tagUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) { + com.linkedin.common.GlobalTags tags = + (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, entityService, new GlobalTags()); - // Datahub allows removing tags & terms it is not familiar with- however these terms must exist to be added back - if (!entityService.exists(labelUrn) && !isRemoving) { - throw new IllegalArgumentException(String.format("Failed to update %s on %s. %s does not exist.", labelUrn, targetUrn, labelUrn)); + if (!tags.hasTags()) { + tags.setTags(new TagAssociationArray()); } + removeTagsIfExists(tags, tagUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), TAGS_ASPECT_NAME, tags, actor, entityService); + } - if ((subResource != null && subResource.length() > 0) || subResourceType != null) { - if (subResource == null || subResource.length() == 0) { - throw new IllegalArgumentException(String.format( - "Failed to update %s on %s. SubResourceType (%s) provided without a subResource.", labelUrn, targetUrn, subResourceType)); + private static MetadataChangeProposal buildRemoveTagsToSubResourceProposal( + List tagUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) { + com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = + (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( + resource.getResourceUrn(), + EDITABLE_SCHEMA_METADATA, + entityService, + new EditableSchemaMetadata()); + EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, resource.getSubResource()); + + if (!editableFieldInfo.hasGlobalTags()) { + editableFieldInfo.setGlobalTags(new GlobalTags()); + } + removeTagsIfExists(editableFieldInfo.getGlobalTags(), tagUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + } + + private static MetadataChangeProposal buildAddTagsToEntityProposal( + List tagUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + com.linkedin.common.GlobalTags tags = + (com.linkedin.common.GlobalTags) getAspectFromEntity(resource.getResourceUrn(), TAGS_ASPECT_NAME, entityService, new GlobalTags()); + + if (!tags.hasTags()) { + tags.setTags(new TagAssociationArray()); + } + addTagsIfNotExists(tags, tagUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), TAGS_ASPECT_NAME, tags, actor, entityService); + } + + private static MetadataChangeProposal buildAddTagsToSubResourceProposal( + List tagUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = + (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( + resource.getResourceUrn(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); + EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, resource.getSubResource()); + + if (!editableFieldInfo.hasGlobalTags()) { + editableFieldInfo.setGlobalTags(new GlobalTags()); + } + + addTagsIfNotExists(editableFieldInfo.getGlobalTags(), tagUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + } + + private static void addTagsIfNotExists(GlobalTags tags, List tagUrns) throws URISyntaxException { + if (!tags.hasTags()) { + tags.setTags(new TagAssociationArray()); + } + + TagAssociationArray tagAssociationArray = tags.getTags(); + + List tagsToAdd = new ArrayList<>(); + for (Urn tagUrn : tagUrns) { + if (tagAssociationArray.stream().anyMatch(association -> association.getTag().equals(tagUrn))) { + continue; } - if (subResourceType == null) { - throw new IllegalArgumentException(String.format( - "Failed to update %s on %s. SubResource (%s) provided without a subResourceType.", labelUrn, targetUrn, subResource)); + tagsToAdd.add(tagUrn); + } + + // Check for no tags to add + if (tagsToAdd.size() == 0) { + return; + } + + for (Urn tagUrn : tagsToAdd) { + TagAssociation newAssociation = new TagAssociation(); + newAssociation.setTag(TagUrn.createFromUrn(tagUrn)); + tagAssociationArray.add(newAssociation); + } + } + + private static MetadataChangeProposal buildAddTermsProposal( + List termUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + if (resource.getSubResource() == null || resource.getSubResource().equals("")) { + // Case 1: Adding terms to a top-level entity + return buildAddTermsToEntityProposal(termUrns, resource, actor, entityService); + } else { + // Case 2: Adding terms to subresource (e.g. schema fields) + return buildAddTermsToSubResourceProposal(termUrns, resource, actor, entityService); + } + } + + private static MetadataChangeProposal buildRemoveTermsProposal( + List termUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + if (resource.getSubResource() == null || resource.getSubResource().equals("")) { + // Case 1: Removing terms from a top-level entity + return buildRemoveTermsToEntityProposal(termUrns, resource, actor, entityService); + } else { + // Case 2: Removing terms from subresource (e.g. schema fields) + return buildRemoveTermsToSubResourceProposal(termUrns, resource, actor, entityService); + } + } + + private static MetadataChangeProposal buildAddTermsToEntityProposal( + List termUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + com.linkedin.common.GlossaryTerms terms = + (com.linkedin.common.GlossaryTerms) getAspectFromEntity(resource.getResourceUrn(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); + terms.setAuditStamp(getAuditStamp(actor)); + + if (!terms.hasTerms()) { + terms.setTerms(new GlossaryTermAssociationArray()); + } + + addTermsIfNotExists(terms, termUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); + } + + private static MetadataChangeProposal buildAddTermsToSubResourceProposal( + List termUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) throws URISyntaxException { + com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = + (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( + resource.getResourceUrn(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); + + EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, resource.getSubResource()); + if (!editableFieldInfo.hasGlossaryTerms()) { + editableFieldInfo.setGlossaryTerms(new GlossaryTerms()); + } + + editableFieldInfo.getGlossaryTerms().setAuditStamp(getAuditStamp(actor)); + + addTermsIfNotExists(editableFieldInfo.getGlossaryTerms(), termUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + } + + private static MetadataChangeProposal buildRemoveTermsToEntityProposal( + List termUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) { + com.linkedin.common.GlossaryTerms terms = + (com.linkedin.common.GlossaryTerms) MutationUtils.getAspectFromEntity( + resource.getResourceUrn(), GLOSSARY_TERM_ASPECT_NAME, entityService, new GlossaryTerms()); + terms.setAuditStamp(getAuditStamp(actor)); + + removeTermsIfExists(terms, termUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), GLOSSARY_TERM_ASPECT_NAME, terms, actor, entityService); + } + + private static MetadataChangeProposal buildRemoveTermsToSubResourceProposal( + List termUrns, + ResourceRefInput resource, + Urn actor, + EntityService entityService + ) { + com.linkedin.schema.EditableSchemaMetadata editableSchemaMetadata = + (com.linkedin.schema.EditableSchemaMetadata) getAspectFromEntity( + resource.getResourceUrn(), EDITABLE_SCHEMA_METADATA, entityService, new EditableSchemaMetadata()); + EditableSchemaFieldInfo editableFieldInfo = getFieldInfoFromSchema(editableSchemaMetadata, resource.getSubResource()); + if (!editableFieldInfo.hasGlossaryTerms()) { + editableFieldInfo.setGlossaryTerms(new GlossaryTerms()); + } + + removeTermsIfExists(editableFieldInfo.getGlossaryTerms(), termUrns); + return buildMetadataChangeProposal(UrnUtils.getUrn(resource.getResourceUrn()), EDITABLE_SCHEMA_METADATA, editableSchemaMetadata, actor, entityService); + } + + private static void addTermsIfNotExists(GlossaryTerms terms, List termUrns) + throws URISyntaxException { + if (!terms.hasTerms()) { + terms.setTerms(new GlossaryTermAssociationArray()); + } + + GlossaryTermAssociationArray termArray = terms.getTerms(); + + List termsToAdd = new ArrayList<>(); + for (Urn termUrn : termUrns) { + if (termArray.stream().anyMatch(association -> association.getUrn().equals(termUrn))) { + continue; } - validateSubresourceExists(targetUrn, subResource, subResourceType, entityService); + termsToAdd.add(termUrn); + } + + // Check for no terms to add + if (termsToAdd.size() == 0) { + return; + } + + for (Urn termUrn : termsToAdd) { + GlossaryTermAssociation newAssociation = new GlossaryTermAssociation(); + newAssociation.setUrn(GlossaryTermUrn.createFromUrn(termUrn)); + termArray.add(newAssociation); + } + } + + private static GlossaryTermAssociationArray removeTermsIfExists(GlossaryTerms terms, List termUrns) { + if (!terms.hasTerms()) { + terms.setTerms(new GlossaryTermAssociationArray()); } + GlossaryTermAssociationArray termAssociationArray = terms.getTerms(); + for (Urn termUrn : termUrns) { + termAssociationArray.removeIf(association -> association.getUrn().equals(termUrn)); + } + return termAssociationArray; + } - return true; + private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { + // TODO: Replace this with a batch ingest proposals endpoint. + for (MetadataChangeProposal change : changes) { + entityService.ingestProposal(change, getAuditStamp(actor)); + } } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java index a0770dd32e4568..2e06cec2686d1f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/OwnerUtils.java @@ -16,10 +16,13 @@ import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.generated.OwnerEntityType; import com.linkedin.datahub.graphql.generated.OwnerInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -36,28 +39,34 @@ public class OwnerUtils { private OwnerUtils() { } - public static void addOwner( - Urn ownerUrn, - OwnershipType type, - Urn resourceUrn, + public static void addOwnersToResources( + List owners, + List resources, Urn actor, EntityService entityService ) { - Ownership ownershipAspect = (Ownership) getAspectFromEntity( - resourceUrn.toString(), - Constants.OWNERSHIP_ASPECT_NAME, - entityService, - new Ownership()); - addOwner(ownershipAspect, ownerUrn, type); - persistAspect(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildAddOwnersProposal(owners, UrnUtils.getUrn(resource.getResourceUrn()), actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); } - public static void addOwners( - List owners, - Urn resourceUrn, + public static void removeOwnersFromResources( + List ownerUrns, + List resources, Urn actor, EntityService entityService ) { + final List changes = new ArrayList<>(); + for (ResourceRefInput resource : resources) { + changes.add(buildRemoveOwnersProposal(ownerUrns, UrnUtils.getUrn(resource.getResourceUrn()), actor, entityService)); + } + ingestChangeProposals(changes, entityService, actor); + } + + + private static MetadataChangeProposal buildAddOwnersProposal(List owners, Urn resourceUrn, Urn actor, EntityService entityService) { Ownership ownershipAspect = (Ownership) getAspectFromEntity( resourceUrn.toString(), Constants.OWNERSHIP_ASPECT_NAME, @@ -66,11 +75,11 @@ public static void addOwners( for (OwnerInput input : owners) { addOwner(ownershipAspect, UrnUtils.getUrn(input.getOwnerUrn()), OwnershipType.valueOf(input.getType().toString())); } - persistAspect(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); + return buildMetadataChangeProposal(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); } - public static void removeOwner( - Urn ownerUrn, + public static MetadataChangeProposal buildRemoveOwnersProposal( + List ownerUrns, Urn resourceUrn, Urn actor, EntityService entityService @@ -81,8 +90,8 @@ public static void removeOwner( entityService, new Ownership()); ownershipAspect.setLastModified(getAuditStamp(actor)); - removeOwner(ownershipAspect, ownerUrn); - persistAspect(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); + removeOwnersIfExists(ownershipAspect, ownerUrns); + return buildMetadataChangeProposal(resourceUrn, Constants.OWNERSHIP_ASPECT_NAME, ownershipAspect, actor, entityService); } private static void addOwner(Ownership ownershipAspect, Urn ownerUrn, OwnershipType type) { @@ -103,13 +112,15 @@ private static void addOwner(Ownership ownershipAspect, Urn ownerUrn, OwnershipT ownershipAspect.setOwners(ownerArray); } - private static void removeOwner(Ownership ownership, Urn ownerUrn) { + private static void removeOwnersIfExists(Ownership ownership, List ownerUrns) { if (!ownership.hasOwners()) { ownership.setOwners(new OwnerArray()); } OwnerArray ownerArray = ownership.getOwners(); - ownerArray.removeIf(owner -> owner.getOwner().equals(ownerUrn)); + for (Urn ownerUrn : ownerUrns) { + ownerArray.removeIf(owner -> owner.getOwner().equals(ownerUrn)); + } } public static boolean isAuthorizedToUpdateOwners(@Nonnull QueryContext context, Urn resourceUrn) { @@ -170,6 +181,26 @@ public static Boolean validateAddInput( return true; } + public static void validateOwner( + Urn ownerUrn, + OwnerEntityType ownerEntityType, + EntityService entityService + ) { + if (OwnerEntityType.CORP_GROUP.equals(ownerEntityType) && !Constants.CORP_GROUP_ENTITY_NAME.equals(ownerUrn.getEntityType())) { + throw new IllegalArgumentException( + String.format("Failed to change ownership for resource(s). Expected a corp group urn, found %s", ownerUrn)); + } + + if (OwnerEntityType.CORP_USER.equals(ownerEntityType) && !Constants.CORP_USER_ENTITY_NAME.equals(ownerUrn.getEntityType())) { + throw new IllegalArgumentException( + String.format("Failed to change ownership for resource(s). Expected a corp user urn, found %s.", ownerUrn)); + } + + if (!entityService.exists(ownerUrn)) { + throw new IllegalArgumentException(String.format("Failed to change ownership for resource(s). Owner with urn %s does not exist.", ownerUrn)); + } + } + public static Boolean validateRemoveInput( Urn resourceUrn, EntityService entityService @@ -179,4 +210,11 @@ public static Boolean validateRemoveInput( } return true; } + + private static void ingestChangeProposals(List changes, EntityService entityService, Urn actor) { + // TODO: Replace this with a batch ingest proposals endpoint. + for (MetadataChangeProposal change : changes) { + entityService.ingestProposal(change, getAuditStamp(actor)); + } + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java index 494f8be18c8861..9df1574312edc6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java @@ -57,7 +57,7 @@ public CompletableFuture get(DataFetchingEnvironmen environment); } - // By default, autocomplete only against the set of Searchable Entity Types. + // By default, autocomplete only against the Default Set of Autocomplete entities return AutocompleteUtils.batchGetAutocompleteResults( AUTO_COMPLETE_ENTITY_TYPES.stream().map(_typeToEntity::get).collect(Collectors.toList()), sanitizedQuery, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java index ade14be28e646f..e4485e25439bdd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java @@ -3,6 +3,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.LineageDirection; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageInput; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults; @@ -14,7 +15,9 @@ import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -57,6 +60,8 @@ public CompletableFuture get(DataFetchingEnvironment final int start = input.getStart() != null ? input.getStart() : DEFAULT_START; final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT; + final List filters = input.getFilters() != null ? input.getFilters() : new ArrayList<>(); + final Integer maxHops = getMaxHops(filters); com.linkedin.metadata.graph.LineageDirection resolvedDirection = com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString()); @@ -64,19 +69,36 @@ public CompletableFuture get(DataFetchingEnvironment try { log.debug( "Executing search across relationships: source urn {}, direction {}, entity types {}, query {}, filters: {}, start: {}, count: {}", - urn, resolvedDirection, input.getTypes(), input.getQuery(), input.getFilters(), start, count); + urn, resolvedDirection, input.getTypes(), input.getQuery(), filters, start, count); return UrnSearchAcrossLineageResultsMapper.map( _entityClient.searchAcrossLineage(urn, resolvedDirection, entityNames, sanitizedQuery, - ResolverUtils.buildFilter(input.getFilters()), null, start, count, + maxHops, ResolverUtils.buildFilter(filters), null, start, count, ResolverUtils.getAuthentication(environment))); } catch (RemoteInvocationException e) { log.error( "Failed to execute search across relationships: source urn {}, direction {}, entity types {}, query {}, filters: {}, start: {}, count: {}", - urn, resolvedDirection, input.getTypes(), input.getQuery(), input.getFilters(), start, count); + urn, resolvedDirection, input.getTypes(), input.getQuery(), filters, start, count); throw new RuntimeException("Failed to execute search across relationships: " + String.format( "source urn %s, direction %s, entity types %s, query %s, filters: %s, start: %s, count: %s", urn, - resolvedDirection, input.getTypes(), input.getQuery(), input.getFilters(), start, count), e); + resolvedDirection, input.getTypes(), input.getQuery(), filters, start, count), e); } }); } + +// Assumption is that filter values for degree are either null, 3+, 2, or 1. + private Integer getMaxHops(List filters) { + Set degreeFilterValues = filters.stream() + .filter(filter -> filter.getField().equals("degree")) + .map(FacetFilterInput::getValue) + .collect(Collectors.toSet()); + Integer maxHops = null; + if (!degreeFilterValues.contains("3+")) { + if (degreeFilterValues.contains("2")) { + maxHops = 2; + } else if (degreeFilterValues.contains("1")) { + maxHops = 1; + } + } + return maxHops; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java new file mode 100644 index 00000000000000..3ffa261cae4409 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolver.java @@ -0,0 +1,79 @@ +package com.linkedin.datahub.graphql.resolvers.tag; + +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateTagInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.key.TagKey; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.tag.TagProperties; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + +/** + * Resolver used for creating a new Tag on DataHub. Requires the CREATE_TAG or MANAGE_TAGS privilege. + */ +@Slf4j +@RequiredArgsConstructor +public class CreateTagResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final CreateTagInput input = bindArgument(environment.getArgument("input"), CreateTagInput.class); + + return CompletableFuture.supplyAsync(() -> { + + if (!AuthorizationUtils.canCreateTags(context)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + try { + // Create the Tag Key + final TagKey key = new TagKey(); + + // Take user provided id OR generate a random UUID for the Tag. + final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + key.setName(id); + + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.TAG_ENTITY_NAME), context.getAuthentication())) { + throw new IllegalArgumentException("This Tag already exists!"); + } + + // Create the MCP + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + proposal.setEntityType(Constants.TAG_ENTITY_NAME); + proposal.setAspectName(Constants.TAG_PROPERTIES_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(mapTagProperties(input))); + proposal.setChangeType(ChangeType.UPSERT); + return _entityClient.ingestProposal(proposal, context.getAuthentication()); + } catch (Exception e) { + log.error("Failed to create Domain with id: {}, name: {}: {}", input.getId(), input.getName(), e.getMessage()); + throw new RuntimeException(String.format("Failed to create Domain with id: %s, name: %s", input.getId(), input.getName()), e); + } + }); + } + + private TagProperties mapTagProperties(final CreateTagInput input) { + final TagProperties result = new TagProperties(); + result.setName(input.getName()); + result.setDescription(input.getDescription(), SetMode.IGNORE_NULL); + return result; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/DeleteTagResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/DeleteTagResolver.java new file mode 100644 index 00000000000000..72b95935838ef5 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/tag/DeleteTagResolver.java @@ -0,0 +1,58 @@ +package com.linkedin.datahub.graphql.resolvers.tag; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; + + +/** + * Resolver responsible for hard deleting a particular DataHub Corp Group + */ +@Slf4j +public class DeleteTagResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public DeleteTagResolver(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final String tagUrn = environment.getArgument("urn"); + final Urn urn = Urn.createFromString(tagUrn); + + return CompletableFuture.supplyAsync(() -> { + + if (AuthorizationUtils.canManageTags(context) || AuthorizationUtils.canDeleteEntity(UrnUtils.getUrn(tagUrn), context)) { + try { + _entityClient.deleteEntity(urn, context.getAuthentication()); + + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync(() -> { + try { + _entityClient.deleteEntityReferences(urn, context.getAuthentication()); + } catch (RemoteInvocationException e) { + log.error(String.format( + "Caught exception while attempting to clear all entity references for Tag with urn %s", urn), e); + } + }); + + return true; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to perform delete against domain with urn %s", tagUrn), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java index 7454d07fc99551..d792771df14d42 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/test/CreateTestResolver.java @@ -8,6 +8,7 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.TestKey; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.test.TestInfo; @@ -34,32 +35,37 @@ public CreateTestResolver(final EntityClient entityClient) { @Override public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); + final CreateTestInput input = bindArgument(environment.getArgument("input"), CreateTestInput.class); return CompletableFuture.supplyAsync(() -> { if (canManageTests(context)) { - final CreateTestInput input = bindArgument(environment.getArgument("input"), CreateTestInput.class); - final MetadataChangeProposal proposal = new MetadataChangeProposal(); + try { - // Create new test - // Since we are creating a new Test, we need to generate a unique UUID. - final UUID uuid = UUID.randomUUID(); - final String uuidStr = input.getId() == null ? uuid.toString() : input.getId(); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); - // Create the Ingestion source key - final TestKey key = new TestKey(); - key.setId(uuidStr); - proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + // Create new test + // Since we are creating a new Test, we need to generate a unique UUID. + final UUID uuid = UUID.randomUUID(); + final String uuidStr = input.getId() == null ? uuid.toString() : input.getId(); - // Create the Test info. - final TestInfo info = mapCreateTestInput(input); - proposal.setEntityType(Constants.TEST_ENTITY_NAME); - proposal.setAspectName(Constants.TEST_INFO_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(info)); - proposal.setChangeType(ChangeType.UPSERT); + // Create the Ingestion source key + final TestKey key = new TestKey(); + key.setId(uuidStr); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + + if (_entityClient.exists(EntityKeyUtils.convertEntityKeyToUrn(key, Constants.TEST_ENTITY_NAME), context.getAuthentication())) { + throw new IllegalArgumentException("This Test already exists!"); + } + + // Create the Test info. + final TestInfo info = mapCreateTestInput(input); + proposal.setEntityType(Constants.TEST_ENTITY_NAME); + proposal.setAspectName(Constants.TEST_INFO_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(info)); + proposal.setChangeType(ChangeType.UPSERT); - try { return _entityClient.ingestProposal(proposal, context.getAuthentication()); } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform update against Test with urn %s", input.toString()), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaBlameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaBlameResolver.java index 005c60fcd32f97..5690a4d1a9e027 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaBlameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaBlameResolver.java @@ -3,7 +3,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.GetSchemaBlameInput; import com.linkedin.datahub.graphql.generated.GetSchemaBlameResult; -import com.linkedin.datahub.graphql.types.timeline.mappers.SchemaFieldBlameMapper; +import com.linkedin.datahub.graphql.types.timeline.mappers.SchemaBlameMapper; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeTransaction; @@ -46,7 +46,7 @@ public CompletableFuture get(final DataFetchingEnvironment Urn datasetUrn = Urn.createFromString(datasetUrnString); List changeTransactionList = _timelineService.getTimeline(datasetUrn, changeCategorySet, startTime, endTime, null, null, false); - return SchemaFieldBlameMapper.map(changeTransactionList, version); + return SchemaBlameMapper.map(changeTransactionList, version); } catch (URISyntaxException u) { log.error( String.format("Failed to list schema blame data, likely due to the Urn %s being invalid", datasetUrnString), diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaVersionListResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaVersionListResolver.java new file mode 100644 index 00000000000000..cfad1395a61a88 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/timeline/GetSchemaVersionListResolver.java @@ -0,0 +1,61 @@ +package com.linkedin.datahub.graphql.resolvers.timeline; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.GetSchemaVersionListInput; +import com.linkedin.datahub.graphql.generated.GetSchemaVersionListResult; +import com.linkedin.datahub.graphql.types.timeline.mappers.SchemaVersionListMapper; +import com.linkedin.metadata.timeline.TimelineService; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeTransaction; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/* +Returns the most recent changes made to each column in a dataset at each dataset version. + */ +@Slf4j +public class GetSchemaVersionListResolver implements DataFetcher> { + private final TimelineService _timelineService; + + public GetSchemaVersionListResolver(TimelineService timelineService) { + _timelineService = timelineService; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final GetSchemaVersionListInput input = + bindArgument(environment.getArgument("input"), GetSchemaVersionListInput.class); + + final String datasetUrnString = input.getDatasetUrn(); + final long startTime = 0; + final long endTime = 0; + + return CompletableFuture.supplyAsync(() -> { + try { + final Set changeCategorySet = new HashSet<>(); + changeCategorySet.add(ChangeCategory.TECHNICAL_SCHEMA); + Urn datasetUrn = Urn.createFromString(datasetUrnString); + List changeTransactionList = + _timelineService.getTimeline(datasetUrn, changeCategorySet, startTime, endTime, null, null, false); + return SchemaVersionListMapper.map(changeTransactionList); + } catch (URISyntaxException u) { + log.error( + String.format("Failed to list schema blame data, likely due to the Urn %s being invalid", datasetUrnString), + u); + return null; + } catch (Exception e) { + log.error("Failed to list schema blame data", e); + return null; + } + }); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserInviteTokenResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserInviteTokenResolver.java new file mode 100644 index 00000000000000..9f8a5ef0e87e49 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserInviteTokenResolver.java @@ -0,0 +1,41 @@ +package com.linkedin.datahub.graphql.resolvers.user; + +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.InviteToken; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.datahub.graphql.authorization.AuthorizationUtils.*; + +/** + * Resolver responsible for creating an invite token that Admins can share with prospective users to create native + * user accounts. + */ +public class CreateNativeUserInviteTokenResolver implements DataFetcher> { + private final NativeUserService _nativeUserService; + + public CreateNativeUserInviteTokenResolver(final NativeUserService nativeUserService) { + _nativeUserService = nativeUserService; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + return CompletableFuture.supplyAsync(() -> { + if (!canManageUserCredentials(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + try { + String inviteToken = _nativeUserService.generateNativeUserInviteToken(context.getAuthentication()); + return new InviteToken(inviteToken); + } catch (Exception e) { + throw new RuntimeException("Failed to generate new invite token"); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserResetTokenResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserResetTokenResolver.java new file mode 100644 index 00000000000000..d02f1a5f786a74 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserResetTokenResolver.java @@ -0,0 +1,52 @@ +package com.linkedin.datahub.graphql.resolvers.user; + +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateNativeUserResetTokenInput; +import com.linkedin.datahub.graphql.generated.ResetToken; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.datahub.graphql.authorization.AuthorizationUtils.*; +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + +/** + * Resolver responsible for creating a password reset token that Admins can share with native users to reset their + * credentials. + */ +public class CreateNativeUserResetTokenResolver implements DataFetcher> { + private final NativeUserService _nativeUserService; + + public CreateNativeUserResetTokenResolver(final NativeUserService nativeUserService) { + _nativeUserService = nativeUserService; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final CreateNativeUserResetTokenInput input = + bindArgument(environment.getArgument("input"), CreateNativeUserResetTokenInput.class); + + final String userUrnString = input.getUserUrn(); + Objects.requireNonNull(userUrnString, "No user urn was provided!"); + + if (!canManageUserCredentials(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + return CompletableFuture.supplyAsync(() -> { + try { + String resetToken = + _nativeUserService.generateNativeUserPasswordResetToken(userUrnString, context.getAuthentication()); + return new ResetToken(resetToken); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to generate password reset token for user: %s", userUrnString)); + } + }); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/GetNativeUserInviteTokenResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/GetNativeUserInviteTokenResolver.java new file mode 100644 index 00000000000000..1d1a329c790b3f --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/GetNativeUserInviteTokenResolver.java @@ -0,0 +1,42 @@ +package com.linkedin.datahub.graphql.resolvers.user; + +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.InviteToken; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.datahub.graphql.authorization.AuthorizationUtils.*; + +/** + * Resolver responsible for getting an existing invite token that Admins can share with prospective users to create + * native user accounts. If the invite token does not already exist, this resolver will create a new one. + */ +public class GetNativeUserInviteTokenResolver implements DataFetcher> { + private final NativeUserService _nativeUserService; + + public GetNativeUserInviteTokenResolver(final NativeUserService nativeUserService) { + _nativeUserService = nativeUserService; + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + return CompletableFuture.supplyAsync(() -> { + if (!canManageUserCredentials(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + try { + String inviteToken = _nativeUserService.getNativeUserInviteToken(context.getAuthentication()); + return new InviteToken(inviteToken); + } catch (Exception e) { + throw new RuntimeException("Failed to generate new invite token"); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/RemoveUserResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/RemoveUserResolver.java index ecc183be0252c0..f77823d47ee9ae 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/RemoveUserResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/user/RemoveUserResolver.java @@ -5,14 +5,17 @@ import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; /** * Resolver responsible for hard deleting a particular DataHub Corp User */ +@Slf4j public class RemoveUserResolver implements DataFetcher> { private final EntityClient _entityClient; @@ -30,6 +33,16 @@ public CompletableFuture get(final DataFetchingEnvironment environment) return CompletableFuture.supplyAsync(() -> { try { _entityClient.deleteEntity(urn, context.getAuthentication()); + + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync(() -> { + try { + _entityClient.deleteEntityReferences(urn, context.getAuthentication()); + } catch (RemoteInvocationException e) { + log.error(String.format("Caught exception while attempting to clear all entity references for user with urn %s", urn), e); + } + }); + return true; } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform delete against user with urn %s", userUrn), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BatchMutableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BatchMutableType.java new file mode 100644 index 00000000000000..3bd8719a37abc4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/BatchMutableType.java @@ -0,0 +1,16 @@ +package com.linkedin.datahub.graphql.types; + +import com.linkedin.datahub.graphql.QueryContext; + +import javax.annotation.Nonnull; +import java.util.List; + +public interface BatchMutableType extends MutableType { + default Class batchInputClass() throws UnsupportedOperationException { + throw new UnsupportedOperationException(this.getClass().getName() + " does not implement batchInputClass method"); + } + + default List batchUpdate(@Nonnull final B[] updateInput, QueryContext context) throws Exception { + throw new UnsupportedOperationException(this.getClass().getName() + " does not implement batchUpdate method"); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/MutableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/MutableType.java index 6b5a349e73d476..94f1200d3a7833 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/MutableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/MutableType.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.types; import com.linkedin.datahub.graphql.QueryContext; + import javax.annotation.Nonnull; /** @@ -9,12 +10,11 @@ * @param : The input type corresponding to the write. */ public interface MutableType { - /** * Returns generated GraphQL class associated with the input type */ - Class inputClass(); + Class inputClass(); /** * Update an entity by urn diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java index 30f154bd2d57a1..c9e2c322ace8df 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java @@ -1,25 +1,24 @@ package com.linkedin.datahub.graphql.types.aspect; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.Aspect; import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMetadataMapper; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.metadata.Constants; import javax.annotation.Nonnull; -public class AspectMapper implements ModelMapper { +public class AspectMapper { public static final AspectMapper INSTANCE = new AspectMapper(); - public static Aspect map(@Nonnull final EnvelopedAspect aspect) { - return INSTANCE.apply(aspect); + public static Aspect map(@Nonnull final EnvelopedAspect aspect, @Nonnull final Urn entityUrn) { + return INSTANCE.apply(aspect, entityUrn); } - @Override - public Aspect apply(@Nonnull final EnvelopedAspect aspect) { + public Aspect apply(@Nonnull final EnvelopedAspect aspect, @Nonnull final Urn entityUrn) { if (Constants.SCHEMA_METADATA_ASPECT_NAME.equals(aspect.getName())) { - return SchemaMetadataMapper.map(aspect); + return SchemaMetadataMapper.map(aspect, entityUrn); } return null; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index 50762d08ab55a5..f3fdfdaa86f9e3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -62,7 +62,7 @@ public List> batchLoad(@Nonnull ListnewResult().data(null).build(); } final EnvelopedAspect aspect = entityResponse.getAspects().get(key.getAspectName()); - return DataFetcherResult.newResult().data(AspectMapper.map(aspect)).build(); + return DataFetcherResult.newResult().data(AspectMapper.map(aspect, entityUrn)).build(); } catch (Exception e) { if (e instanceof RestLiResponseException) { // if no aspect is found, restli will return a 404 rather than null diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java index d88e5cc2c19648..e1d81bb31f4712 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java @@ -142,4 +142,4 @@ private static AssertionStdParameter mapParameter(final com.linkedin.assertion.A private AssertionMapper() { } -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java index 46c4806e71e646..5da31b2db66e01 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/mappers/ChartMapper.java @@ -8,6 +8,7 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.AccessLevel; import com.linkedin.datahub.graphql.generated.Chart; @@ -20,7 +21,6 @@ import com.linkedin.datahub.graphql.generated.Container; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.Dataset; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; @@ -28,8 +28,9 @@ import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -56,24 +57,25 @@ public static Chart map(@Nonnull final EntityResponse entityResponse) { @Override public Chart apply(@Nonnull final EntityResponse entityResponse) { final Chart result = new Chart(); + Urn entityUrn = entityResponse.getUrn(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.CHART); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(CHART_KEY_ASPECT_NAME, this::mapChartKey); - mappingHelper.mapToResult(CHART_INFO_ASPECT_NAME, this::mapChartInfo); + mappingHelper.mapToResult(CHART_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapChartInfo(entity, dataMap, entityUrn)); mappingHelper.mapToResult(CHART_QUERY_ASPECT_NAME, this::mapChartQuery); mappingHelper.mapToResult(EDITABLE_CHART_PROPERTIES_ASPECT_NAME, this::mapEditableChartProperties); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (chart, dataMap) -> - chart.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + chart.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (chart, dataMap) -> chart.setStatus(StatusMapper.map(new Status(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (dataset, dataMap) -> this.mapGlobalTags(dataset, dataMap, entityUrn)); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (chart, dataMap) -> chart.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (chart, dataMap) -> - chart.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + chart.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (chart, dataMap) -> @@ -95,16 +97,16 @@ private void mapChartKey(@Nonnull Chart chart, @Nonnull DataMap dataMap) { .setPlatformName(gmsKey.getDashboardTool()), DATA_PLATFORM_ENTITY_NAME).toString()).build()); } - private void mapChartInfo(@Nonnull Chart chart, @Nonnull DataMap dataMap) { + private void mapChartInfo(@Nonnull Chart chart, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { final com.linkedin.chart.ChartInfo gmsChartInfo = new com.linkedin.chart.ChartInfo(dataMap); - chart.setInfo(mapInfo(gmsChartInfo)); - chart.setProperties(mapChartInfoToProperties(gmsChartInfo)); + chart.setInfo(mapInfo(gmsChartInfo, entityUrn)); + chart.setProperties(mapChartInfoToProperties(gmsChartInfo, entityUrn)); } /** * Maps GMS {@link com.linkedin.chart.ChartInfo} to deprecated GraphQL {@link ChartInfo} */ - private ChartInfo mapInfo(final com.linkedin.chart.ChartInfo info) { + private ChartInfo mapInfo(final com.linkedin.chart.ChartInfo info, @Nonnull Urn entityUrn) { final ChartInfo result = new ChartInfo(); result.setDescription(info.getDescription()); result.setName(info.getTitle()); @@ -136,7 +138,7 @@ private ChartInfo mapInfo(final com.linkedin.chart.ChartInfo info) { result.setExternalUrl(info.getChartUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } return result; } @@ -144,7 +146,7 @@ private ChartInfo mapInfo(final com.linkedin.chart.ChartInfo info) { /** * Maps GMS {@link com.linkedin.chart.ChartInfo} to new GraphQL {@link ChartProperties} */ - private ChartProperties mapChartInfoToProperties(final com.linkedin.chart.ChartInfo info) { + private ChartProperties mapChartInfoToProperties(final com.linkedin.chart.ChartInfo info, @Nonnull Urn entityUrn) { final ChartProperties result = new ChartProperties(); result.setDescription(info.getDescription()); result.setName(info.getTitle()); @@ -168,7 +170,7 @@ private ChartProperties mapChartInfoToProperties(final com.linkedin.chart.ChartI result.setExternalUrl(info.getChartUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } return result; } @@ -192,8 +194,8 @@ private void mapEditableChartProperties(@Nonnull Chart chart, @Nonnull DataMap d chart.setEditableProperties(chartEditableProperties); } - private void mapGlobalTags(@Nonnull Chart chart, @Nonnull DataMap dataMap) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap)); + private void mapGlobalTags(@Nonnull Chart chart, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); chart.setGlobalTags(globalTags); chart.setTags(globalTags); } @@ -209,11 +211,6 @@ private void mapContainers(@Nonnull Chart chart, @Nonnull DataMap dataMap) { private void mapDomains(@Nonnull Chart chart, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); - // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - chart.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + chart.setDomain(DomainAssociationMapper.map(domains, chart.getUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/CustomPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/CustomPropertiesMapper.java new file mode 100644 index 00000000000000..50e4846611a9b0 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/CustomPropertiesMapper.java @@ -0,0 +1,36 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.CustomPropertiesEntry; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. + * + * To be replaced by auto-generated mappers implementations + */ +public class CustomPropertiesMapper { + + public static final CustomPropertiesMapper INSTANCE = new CustomPropertiesMapper(); + + public static List map(@Nonnull final Map input, @Nonnull Urn urn) { + return INSTANCE.apply(input, urn); + } + + public List apply(@Nonnull final Map input, @Nonnull Urn urn) { + List results = new ArrayList<>(); + for (String key : input.keySet()) { + final CustomPropertiesEntry entry = new CustomPropertiesEntry(); + entry.setKey(key); + entry.setValue(input.get(key)); + entry.setAssociatedUrn(urn.toString()); + results.add(entry); + } + return results; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java index c6fb2044c1773f..d66c5fd09b8f42 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnerMapper.java @@ -1,10 +1,10 @@ package com.linkedin.datahub.graphql.types.common.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.CorpGroup; import com.linkedin.datahub.graphql.generated.Owner; import com.linkedin.datahub.graphql.generated.OwnershipType; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import javax.annotation.Nonnull; @@ -13,16 +13,15 @@ * * To be replaced by auto-generated mappers implementations */ -public class OwnerMapper implements ModelMapper { +public class OwnerMapper { public static final OwnerMapper INSTANCE = new OwnerMapper(); - public static Owner map(@Nonnull final com.linkedin.common.Owner owner) { - return INSTANCE.apply(owner); + public static Owner map(@Nonnull final com.linkedin.common.Owner owner, @Nonnull final Urn entityUrn) { + return INSTANCE.apply(owner, entityUrn); } - @Override - public Owner apply(@Nonnull final com.linkedin.common.Owner owner) { + public Owner apply(@Nonnull final com.linkedin.common.Owner owner, @Nonnull final Urn entityUrn) { final Owner result = new Owner(); result.setType(Enum.valueOf(OwnershipType.class, owner.getType().toString())); if (owner.getOwner().getEntityType().equals("corpuser")) { @@ -37,6 +36,7 @@ public Owner apply(@Nonnull final com.linkedin.common.Owner owner) { if (owner.hasSource()) { result.setSource(OwnershipSourceMapper.map(owner.getSource())); } + result.setAssociatedUrn(entityUrn.toString()); return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnershipMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnershipMapper.java index 8c80bac2bb1f58..6614cfb28a4784 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnershipMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/OwnershipMapper.java @@ -1,7 +1,7 @@ package com.linkedin.datahub.graphql.types.common.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.Ownership; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import javax.annotation.Nonnull; import java.util.stream.Collectors; @@ -11,21 +11,20 @@ * * To be replaced by auto-generated mappers implementations */ -public class OwnershipMapper implements ModelMapper { +public class OwnershipMapper { public static final OwnershipMapper INSTANCE = new OwnershipMapper(); - public static Ownership map(@Nonnull final com.linkedin.common.Ownership ownership) { - return INSTANCE.apply(ownership); + public static Ownership map(@Nonnull final com.linkedin.common.Ownership ownership, @Nonnull final Urn entityUrn) { + return INSTANCE.apply(ownership, entityUrn); } - @Override - public Ownership apply(@Nonnull final com.linkedin.common.Ownership ownership) { + public Ownership apply(@Nonnull final com.linkedin.common.Ownership ownership, @Nonnull final Urn entityUrn) { final Ownership result = new Ownership(); result.setLastModified(AuditStampMapper.map(ownership.getLastModified())); result.setOwners(ownership.getOwners() .stream() - .map(OwnerMapper::map) + .map(owner -> OwnerMapper.map(owner, entityUrn)) .collect(Collectors.toList())); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SiblingsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SiblingsMapper.java new file mode 100644 index 00000000000000..942171017cea4a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/SiblingsMapper.java @@ -0,0 +1,32 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.datahub.graphql.generated.SiblingProperties; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + + +/** + * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. + * + * To be replaced by auto-generated mappers implementations + */ +public class SiblingsMapper implements ModelMapper { + + public static final SiblingsMapper INSTANCE = new SiblingsMapper(); + + public static SiblingProperties map(@Nonnull final com.linkedin.common.Siblings siblings) { + return INSTANCE.apply(siblings); + } + + @Override + public SiblingProperties apply(@Nonnull final com.linkedin.common.Siblings siblings) { + final SiblingProperties result = new SiblingProperties(); + result.setIsPrimary(siblings.isPrimary()); + result.setSiblings(siblings.getSiblings() + .stream() + .map(UrnToEntityMapper::map) + .collect(Collectors.toList())); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/StringMapMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/StringMapMapper.java index 67754ad6aab01d..32c49a20104142 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/StringMapMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/StringMapMapper.java @@ -1,13 +1,12 @@ package com.linkedin.datahub.graphql.types.common.mappers; - import com.linkedin.datahub.graphql.generated.StringMapEntry; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; - -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; + /** * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java index e13acd06e2181f..013f074cdca84c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java @@ -13,13 +13,13 @@ import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.Container; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.domain.Domains; @@ -56,7 +56,7 @@ public static Container map(final EntityResponse entityResponse) { final EnvelopedAspect envelopedContainerProperties = aspects.get(Constants.CONTAINER_PROPERTIES_ASPECT_NAME); if (envelopedContainerProperties != null) { - result.setProperties(mapContainerProperties(new ContainerProperties(envelopedContainerProperties.getValue().data()))); + result.setProperties(mapContainerProperties(new ContainerProperties(envelopedContainerProperties.getValue().data()), entityUrn)); } final EnvelopedAspect envelopedEditableContainerProperties = aspects.get(Constants.CONTAINER_EDITABLE_PROPERTIES_ASPECT_NAME); @@ -66,18 +66,18 @@ public static Container map(final EntityResponse entityResponse) { final EnvelopedAspect envelopedOwnership = aspects.get(Constants.OWNERSHIP_ASPECT_NAME); if (envelopedOwnership != null) { - result.setOwnership(OwnershipMapper.map(new Ownership(envelopedOwnership.getValue().data()))); + result.setOwnership(OwnershipMapper.map(new Ownership(envelopedOwnership.getValue().data()), entityUrn)); } final EnvelopedAspect envelopedTags = aspects.get(Constants.GLOBAL_TAGS_ASPECT_NAME); if (envelopedTags != null) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(envelopedTags.getValue().data())); + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(envelopedTags.getValue().data()), entityUrn); result.setTags(globalTags); } final EnvelopedAspect envelopedTerms = aspects.get(Constants.GLOSSARY_TERMS_ASPECT_NAME); if (envelopedTerms != null) { - result.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(envelopedTerms.getValue().data()))); + result.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(envelopedTerms.getValue().data()), entityUrn)); } final EnvelopedAspect envelopedInstitutionalMemory = aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME); @@ -104,11 +104,7 @@ public static Container map(final EntityResponse entityResponse) { if (envelopedDomains != null) { final Domains domains = new Domains(envelopedDomains.getValue().data()); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - result.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + result.setDomain(DomainAssociationMapper.map(domains, entityUrn.toString())); } final EnvelopedAspect envelopedDeprecation = aspects.get(Constants.DEPRECATION_ASPECT_NAME); @@ -119,7 +115,7 @@ public static Container map(final EntityResponse entityResponse) { return result; } - private static com.linkedin.datahub.graphql.generated.ContainerProperties mapContainerProperties(final ContainerProperties gmsProperties) { + private static com.linkedin.datahub.graphql.generated.ContainerProperties mapContainerProperties(final ContainerProperties gmsProperties, Urn entityUrn) { final com.linkedin.datahub.graphql.generated.ContainerProperties propertiesResult = new com.linkedin.datahub.graphql.generated.ContainerProperties(); propertiesResult.setName(gmsProperties.getName()); propertiesResult.setDescription(gmsProperties.getDescription()); @@ -127,7 +123,7 @@ private static com.linkedin.datahub.graphql.generated.ContainerProperties mapCon propertiesResult.setExternalUrl(gmsProperties.getExternalUrl().toString()); } if (gmsProperties.hasCustomProperties()) { - propertiesResult.setCustomProperties(StringMapMapper.map(gmsProperties.getCustomProperties())); + propertiesResult.setCustomProperties(CustomPropertiesMapper.map(gmsProperties.getCustomProperties(), entityUrn)); } return propertiesResult; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java index c9b8bcafd80e21..0fb1b66c644d78 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java @@ -1,6 +1,8 @@ package com.linkedin.datahub.graphql.types.corpgroup.mappers; +import com.linkedin.common.Origin; import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.CorpGroup; import com.linkedin.datahub.graphql.generated.EntityType; @@ -33,6 +35,8 @@ public static CorpGroup map(@Nonnull final EntityResponse entityResponse) { @Override public CorpGroup apply(@Nonnull final EntityResponse entityResponse) { final CorpGroup result = new CorpGroup(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.CORP_GROUP); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); @@ -40,7 +44,15 @@ public CorpGroup apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(CORP_GROUP_KEY_ASPECT_NAME, this::mapCorpGroupKey); mappingHelper.mapToResult(CORP_GROUP_INFO_ASPECT_NAME, this::mapCorpGroupInfo); mappingHelper.mapToResult(CORP_GROUP_EDITABLE_INFO_ASPECT_NAME, this::mapCorpGroupEditableInfo); - mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, this::mapOwnership); + mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (entity, dataMap) -> this.mapOwnership(entity, dataMap, entityUrn)); + if (aspectMap.containsKey(ORIGIN_ASPECT_NAME)) { + mappingHelper.mapToResult(ORIGIN_ASPECT_NAME, this::mapEntityOriginType); + } else { + com.linkedin.datahub.graphql.generated.Origin mappedGroupOrigin = + new com.linkedin.datahub.graphql.generated.Origin(); + mappedGroupOrigin.setType(com.linkedin.datahub.graphql.generated.OriginType.UNKNOWN); + result.setOrigin(mappedGroupOrigin); + } return mappingHelper.getResult(); } @@ -59,7 +71,23 @@ private void mapCorpGroupEditableInfo(@Nonnull CorpGroup corpGroup, @Nonnull Dat corpGroup.setEditableProperties(CorpGroupEditablePropertiesMapper.map(new CorpGroupEditableInfo(dataMap))); } - private void mapOwnership(@Nonnull CorpGroup corpGroup, @Nonnull DataMap dataMap) { - corpGroup.setOwnership(OwnershipMapper.map(new Ownership(dataMap))); + private void mapOwnership(@Nonnull CorpGroup corpGroup, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { + corpGroup.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn)); + } + + private void mapEntityOriginType(@Nonnull CorpGroup corpGroup, @Nonnull DataMap dataMap) { + Origin groupOrigin = new Origin(dataMap); + com.linkedin.datahub.graphql.generated.Origin mappedGroupOrigin = + new com.linkedin.datahub.graphql.generated.Origin(); + if (groupOrigin.hasType()) { + mappedGroupOrigin.setType( + com.linkedin.datahub.graphql.generated.OriginType.valueOf(groupOrigin.getType().toString())); + } else { + mappedGroupOrigin.setType(com.linkedin.datahub.graphql.generated.OriginType.UNKNOWN); + } + if (groupOrigin.hasExternalType()) { + mappedGroupOrigin.setExternalType(groupOrigin.getExternalType()); + } + corpGroup.setOrigin(mappedGroupOrigin); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java index 28f661fa997f0e..ac91970f62847d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.types.corpuser.mappers; import com.linkedin.common.GlobalTags; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.generated.CorpUser; @@ -10,6 +11,7 @@ import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.identity.CorpUserCredentials; import com.linkedin.identity.CorpUserEditableInfo; import com.linkedin.identity.CorpUserInfo; import com.linkedin.identity.CorpUserStatus; @@ -35,6 +37,8 @@ public static CorpUser map(@Nonnull final EntityResponse entityResponse) { @Override public CorpUser apply(@Nonnull final EntityResponse entityResponse) { final CorpUser result = new CorpUser(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.CORP_USER); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); @@ -44,9 +48,10 @@ public CorpUser apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(CORP_USER_EDITABLE_INFO_ASPECT_NAME, (corpUser, dataMap) -> corpUser.setEditableProperties(CorpUserEditableInfoMapper.map(new CorpUserEditableInfo(dataMap)))); mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (corpUser, dataMap) -> - corpUser.setGlobalTags(GlobalTagsMapper.map(new GlobalTags(dataMap)))); - mappingHelper.mapToResult(CORP_USER_STATUS_ASPECT_NAME, (corpUser, dataMap) -> - corpUser.setStatus(CorpUserStatusMapper.map(new CorpUserStatus(dataMap)))); + corpUser.setGlobalTags(GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn))); + mappingHelper.mapToResult(CORP_USER_STATUS_ASPECT_NAME, + (corpUser, dataMap) -> corpUser.setStatus(CorpUserStatusMapper.map(new CorpUserStatus(dataMap)))); + mappingHelper.mapToResult(CORP_USER_CREDENTIALS_ASPECT_NAME, this::mapIsNativeUser); return mappingHelper.getResult(); } @@ -60,4 +65,11 @@ private void mapCorpUserInfo(@Nonnull CorpUser corpUser, @Nonnull DataMap dataMa corpUser.setProperties(CorpUserPropertiesMapper.map(corpUserInfo)); corpUser.setInfo(CorpUserInfoMapper.map(corpUserInfo)); } + + private void mapIsNativeUser(@Nonnull CorpUser corpUser, @Nonnull DataMap dataMap) { + CorpUserCredentials corpUserCredentials = new CorpUserCredentials(dataMap); + boolean isNativeUser = + corpUserCredentials != null && corpUserCredentials.hasSalt() && corpUserCredentials.hasHashedPassword(); + corpUser.setIsNativeUser(isNativeUser); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java index 2f4dc0ff5f97b8..d4a39d1c92263c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java @@ -7,6 +7,7 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.dashboard.EditableDashboardProperties; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.AccessLevel; @@ -17,7 +18,6 @@ import com.linkedin.datahub.graphql.generated.DashboardInfo; import com.linkedin.datahub.graphql.generated.DashboardProperties; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; @@ -25,8 +25,9 @@ import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -53,26 +54,28 @@ public static Dashboard map(@Nonnull final EntityResponse entityResponse) { @Override public Dashboard apply(@Nonnull final EntityResponse entityResponse) { final Dashboard result = new Dashboard(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.DASHBOARD); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(DASHBOARD_KEY_ASPECT_NAME, this::mapDashboardKey); - mappingHelper.mapToResult(DASHBOARD_INFO_ASPECT_NAME, this::mapDashboardInfo); + mappingHelper.mapToResult(DASHBOARD_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapDashboardInfo(entity, dataMap, entityUrn)); mappingHelper.mapToResult(EDITABLE_DASHBOARD_PROPERTIES_ASPECT_NAME, this::mapEditableDashboardProperties); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (dashboard, dataMap) -> - dashboard.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + dashboard.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (dashboard, dataMap) -> dashboard.setStatus(StatusMapper.map(new Status(dataMap)))); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dashboard, dataMap) -> dashboard.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (dashboard, dataMap) -> - dashboard.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + dashboard.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (dashboard, dataMap) -> dashboard.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (dataset, dataMap) -> this.mapGlobalTags(dataset, dataMap, entityUrn)); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); @@ -90,16 +93,16 @@ private void mapDashboardKey(@Nonnull Dashboard dashboard, @Nonnull DataMap data .setPlatformName(gmsKey.getDashboardTool()), DATA_PLATFORM_ENTITY_NAME).toString()).build()); } - private void mapDashboardInfo(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMap) { + private void mapDashboardInfo(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMap, Urn entityUrn) { final com.linkedin.dashboard.DashboardInfo gmsDashboardInfo = new com.linkedin.dashboard.DashboardInfo(dataMap); - dashboard.setInfo(mapInfo(gmsDashboardInfo)); - dashboard.setProperties(mapDashboardInfoToProperties(gmsDashboardInfo)); + dashboard.setInfo(mapInfo(gmsDashboardInfo, entityUrn)); + dashboard.setProperties(mapDashboardInfoToProperties(gmsDashboardInfo, entityUrn)); } /** * Maps GMS {@link com.linkedin.dashboard.DashboardInfo} to deprecated GraphQL {@link DashboardInfo} */ - private DashboardInfo mapInfo(final com.linkedin.dashboard.DashboardInfo info) { + private DashboardInfo mapInfo(final com.linkedin.dashboard.DashboardInfo info, Urn entityUrn) { final DashboardInfo result = new DashboardInfo(); result.setDescription(info.getDescription()); result.setName(info.getTitle()); @@ -116,7 +119,7 @@ private DashboardInfo mapInfo(final com.linkedin.dashboard.DashboardInfo info) { result.setExternalUrl(info.getDashboardUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } if (info.hasAccess()) { result.setAccess(AccessLevel.valueOf(info.getAccess().toString())); @@ -132,7 +135,7 @@ private DashboardInfo mapInfo(final com.linkedin.dashboard.DashboardInfo info) { /** * Maps GMS {@link com.linkedin.dashboard.DashboardInfo} to new GraphQL {@link DashboardProperties} */ - private DashboardProperties mapDashboardInfoToProperties(final com.linkedin.dashboard.DashboardInfo info) { + private DashboardProperties mapDashboardInfoToProperties(final com.linkedin.dashboard.DashboardInfo info, Urn entityUrn) { final DashboardProperties result = new DashboardProperties(); result.setDescription(info.getDescription()); result.setName(info.getTitle()); @@ -145,7 +148,7 @@ private DashboardProperties mapDashboardInfoToProperties(final com.linkedin.dash result.setExternalUrl(info.getDashboardUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } if (info.hasAccess()) { result.setAccess(AccessLevel.valueOf(info.getAccess().toString())); @@ -165,8 +168,8 @@ private void mapEditableDashboardProperties(@Nonnull Dashboard dashboard, @Nonnu dashboard.setEditableProperties(dashboardEditableProperties); } - private void mapGlobalTags(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMap) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap)); + private void mapGlobalTags(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); dashboard.setGlobalTags(globalTags); dashboard.setTags(globalTags); } @@ -182,11 +185,6 @@ private void mapContainers(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMa private void mapDomains(@Nonnull Dashboard dashboard, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); - // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - dashboard.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + dashboard.setDomain(DomainAssociationMapper.map(domains, dashboard.getUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUsageMetricMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUsageMetricMapper.java new file mode 100644 index 00000000000000..d257aef4be565e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUsageMetricMapper.java @@ -0,0 +1,34 @@ +package com.linkedin.datahub.graphql.types.dashboard.mappers; + +import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics; +import com.linkedin.datahub.graphql.types.mappers.TimeSeriesAspectMapper; +import com.linkedin.metadata.aspect.EnvelopedAspect; +import com.linkedin.metadata.utils.GenericRecordUtils; +import javax.annotation.Nonnull; + + +public class DashboardUsageMetricMapper implements TimeSeriesAspectMapper { + + public static final DashboardUsageMetricMapper INSTANCE = new DashboardUsageMetricMapper(); + + public static DashboardUsageMetrics map(@Nonnull final EnvelopedAspect envelopedAspect) { + return INSTANCE.apply(envelopedAspect); + } + + @Override + public DashboardUsageMetrics apply(EnvelopedAspect envelopedAspect) { + com.linkedin.dashboard.DashboardUsageStatistics gmsDashboardUsageStatistics = + GenericRecordUtils.deserializeAspect(envelopedAspect.getAspect().getValue(), + envelopedAspect.getAspect().getContentType(), com.linkedin.dashboard.DashboardUsageStatistics.class); + + final com.linkedin.datahub.graphql.generated.DashboardUsageMetrics dashboardUsageMetrics = + new com.linkedin.datahub.graphql.generated.DashboardUsageMetrics(); + dashboardUsageMetrics.setLastViewed(gmsDashboardUsageStatistics.getLastViewedAt()); + dashboardUsageMetrics.setViewsCount(gmsDashboardUsageStatistics.getViewsCount()); + dashboardUsageMetrics.setExecutionsCount(gmsDashboardUsageStatistics.getExecutionsCount()); + dashboardUsageMetrics.setFavoritesCount(gmsDashboardUsageStatistics.getFavoritesCount()); + dashboardUsageMetrics.setTimestampMillis(gmsDashboardUsageStatistics.getTimestampMillis()); + + return dashboardUsageMetrics; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java index 3fabafe8be0de0..1db56d66180285 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java @@ -7,21 +7,22 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.DataFlow; import com.linkedin.datahub.graphql.generated.DataFlowEditableProperties; import com.linkedin.datahub.graphql.generated.DataFlowInfo; import com.linkedin.datahub.graphql.generated.DataFlowProperties; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -50,20 +51,22 @@ public DataFlow apply(@Nonnull final EntityResponse entityResponse) { final DataFlow result = new DataFlow(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.DATA_FLOW); + Urn entityUrn = entityResponse.getUrn(); + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(DATA_FLOW_KEY_ASPECT_NAME, this::mapKey); - mappingHelper.mapToResult(DATA_FLOW_INFO_ASPECT_NAME, this::mapInfo); + mappingHelper.mapToResult(DATA_FLOW_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapInfo(entity, dataMap, entityUrn)); mappingHelper.mapToResult(EDITABLE_DATA_FLOW_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (dataFlow, dataMap) -> - dataFlow.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + dataFlow.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (dataFlow, dataMap) -> dataFlow.setStatus(StatusMapper.map(new Status(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (dataFlow, dataMap) -> this.mapGlobalTags(dataFlow, dataMap, entityUrn)); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataFlow, dataMap) -> dataFlow.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (dataFlow, dataMap) -> - dataFlow.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + dataFlow.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (dataFlow, dataMap) -> dataFlow.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); @@ -85,16 +88,16 @@ private void mapKey(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) { .setPlatformName(gmsKey.getOrchestrator()), DATA_PLATFORM_ENTITY_NAME).toString()).build()); } - private void mapInfo(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) { + private void mapInfo(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap, Urn entityUrn) { final com.linkedin.datajob.DataFlowInfo gmsDataFlowInfo = new com.linkedin.datajob.DataFlowInfo(dataMap); - dataFlow.setInfo(mapDataFlowInfo(gmsDataFlowInfo)); - dataFlow.setProperties(mapDataFlowInfoToProperties(gmsDataFlowInfo)); + dataFlow.setInfo(mapDataFlowInfo(gmsDataFlowInfo, entityUrn)); + dataFlow.setProperties(mapDataFlowInfoToProperties(gmsDataFlowInfo, entityUrn)); } /** * Maps GMS {@link com.linkedin.datajob.DataFlowInfo} to deprecated GraphQL {@link DataFlowInfo} */ - private DataFlowInfo mapDataFlowInfo(final com.linkedin.datajob.DataFlowInfo info) { + private DataFlowInfo mapDataFlowInfo(final com.linkedin.datajob.DataFlowInfo info, Urn entityUrn) { final DataFlowInfo result = new DataFlowInfo(); result.setName(info.getName()); result.setDescription(info.getDescription()); @@ -103,7 +106,7 @@ private DataFlowInfo mapDataFlowInfo(final com.linkedin.datajob.DataFlowInfo inf result.setExternalUrl(info.getExternalUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } return result; } @@ -111,7 +114,7 @@ private DataFlowInfo mapDataFlowInfo(final com.linkedin.datajob.DataFlowInfo inf /** * Maps GMS {@link com.linkedin.datajob.DataFlowInfo} to new GraphQL {@link DataFlowProperties} */ - private DataFlowProperties mapDataFlowInfoToProperties(final com.linkedin.datajob.DataFlowInfo info) { + private DataFlowProperties mapDataFlowInfoToProperties(final com.linkedin.datajob.DataFlowInfo info, Urn entityUrn) { final DataFlowProperties result = new DataFlowProperties(); result.setName(info.getName()); result.setDescription(info.getDescription()); @@ -120,7 +123,7 @@ private DataFlowProperties mapDataFlowInfoToProperties(final com.linkedin.datajo result.setExternalUrl(info.getExternalUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } return result; } @@ -132,8 +135,8 @@ private void mapEditableProperties(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataFlow.setEditableProperties(dataFlowEditableProperties); } - private void mapGlobalTags(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap)); + private void mapGlobalTags(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); dataFlow.setGlobalTags(globalTags); dataFlow.setTags(globalTags); } @@ -141,10 +144,6 @@ private void mapGlobalTags(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) private void mapDomains(@Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - dataFlow.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + dataFlow.setDomain(DomainAssociationMapper.map(domains, dataFlow.getUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java index 7b32da0ecc245d..50788f748fbcdf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java @@ -8,6 +8,7 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.DataFlow; import com.linkedin.datahub.graphql.generated.DataJob; @@ -16,14 +17,14 @@ import com.linkedin.datahub.graphql.generated.DataJobInputOutput; import com.linkedin.datahub.graphql.generated.DataJobProperties; import com.linkedin.datahub.graphql.generated.Dataset; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -48,6 +49,8 @@ public static DataJob map(@Nonnull final EntityResponse entityResponse) { @Override public DataJob apply(@Nonnull final EntityResponse entityResponse) { final DataJob result = new DataJob(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.DATA_JOB); @@ -59,8 +62,8 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) { result.setJobId(gmsKey.getJobId()); } else if (DATA_JOB_INFO_ASPECT_NAME.equals(name)) { final com.linkedin.datajob.DataJobInfo gmsDataJobInfo = new com.linkedin.datajob.DataJobInfo(data); - result.setInfo(mapDataJobInfo(gmsDataJobInfo)); - result.setProperties(mapDataJobInfoToProperties(gmsDataJobInfo)); + result.setInfo(mapDataJobInfo(gmsDataJobInfo, entityUrn)); + result.setProperties(mapDataJobInfoToProperties(gmsDataJobInfo, entityUrn)); } else if (DATA_JOB_INPUT_OUTPUT_ASPECT_NAME.equals(name)) { final com.linkedin.datajob.DataJobInputOutput gmsDataJobInputOutput = new com.linkedin.datajob.DataJobInputOutput(data); result.setInputOutput(mapDataJobInputOutput(gmsDataJobInputOutput)); @@ -70,25 +73,21 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) { dataJobEditableProperties.setDescription(editableDataJobProperties.getDescription()); result.setEditableProperties(dataJobEditableProperties); } else if (OWNERSHIP_ASPECT_NAME.equals(name)) { - result.setOwnership(OwnershipMapper.map(new Ownership(data))); + result.setOwnership(OwnershipMapper.map(new Ownership(data), entityUrn)); } else if (STATUS_ASPECT_NAME.equals(name)) { result.setStatus(StatusMapper.map(new Status(data))); } else if (GLOBAL_TAGS_ASPECT_NAME.equals(name)) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(data)); + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(data), entityUrn); result.setGlobalTags(globalTags); result.setTags(globalTags); } else if (INSTITUTIONAL_MEMORY_ASPECT_NAME.equals(name)) { result.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(data))); } else if (GLOSSARY_TERMS_ASPECT_NAME.equals(name)) { - result.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(data))); + result.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(data), entityUrn)); } else if (DOMAINS_ASPECT_NAME.equals(name)) { final Domains domains = new Domains(data); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - result.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + result.setDomain(DomainAssociationMapper.map(domains, entityUrn.toString())); } else if (DEPRECATION_ASPECT_NAME.equals(name)) { result.setDeprecation(DeprecationMapper.map(new Deprecation(data))); } else if (DATA_PLATFORM_INSTANCE_ASPECT_NAME.equals(name)) { @@ -102,7 +101,7 @@ public DataJob apply(@Nonnull final EntityResponse entityResponse) { /** * Maps GMS {@link com.linkedin.datajob.DataJobInfo} to deprecated GraphQL {@link DataJobInfo} */ - private DataJobInfo mapDataJobInfo(final com.linkedin.datajob.DataJobInfo info) { + private DataJobInfo mapDataJobInfo(final com.linkedin.datajob.DataJobInfo info, Urn entityUrn) { final DataJobInfo result = new DataJobInfo(); result.setName(info.getName()); result.setDescription(info.getDescription()); @@ -110,7 +109,7 @@ private DataJobInfo mapDataJobInfo(final com.linkedin.datajob.DataJobInfo info) result.setExternalUrl(info.getExternalUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } return result; } @@ -118,7 +117,7 @@ private DataJobInfo mapDataJobInfo(final com.linkedin.datajob.DataJobInfo info) /** * Maps GMS {@link com.linkedin.datajob.DataJobInfo} to new GraphQL {@link DataJobProperties} */ - private DataJobProperties mapDataJobInfoToProperties(final com.linkedin.datajob.DataJobInfo info) { + private DataJobProperties mapDataJobInfoToProperties(final com.linkedin.datajob.DataJobInfo info, Urn entityUrn) { final DataJobProperties result = new DataJobProperties(); result.setName(info.getName()); result.setDescription(info.getDescription()); @@ -126,7 +125,7 @@ private DataJobProperties mapDataJobInfoToProperties(final com.linkedin.datajob. result.setExternalUrl(info.getExternalUrl().toString()); } if (info.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(info.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(info.getCustomProperties(), entityUrn)); } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index ca123bb43c063e..87b96f91aeda6e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -11,18 +11,19 @@ import com.linkedin.datahub.graphql.authorization.ConjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.authorization.DisjunctivePrivilegeGroup; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.DatasetUpdateInput; +import com.linkedin.datahub.graphql.generated.Dataset; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.AutoCompleteResults; -import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.BrowseResults; -import com.linkedin.datahub.graphql.generated.Dataset; -import com.linkedin.datahub.graphql.generated.DatasetUpdateInput; +import com.linkedin.datahub.graphql.generated.BrowsePath; import com.linkedin.datahub.graphql.generated.Entity; -import com.linkedin.datahub.graphql.generated.EntityType; -import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.generated.BatchDatasetUpdateInput; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.BatchMutableType; import com.linkedin.datahub.graphql.types.BrowsableEntityType; -import com.linkedin.datahub.graphql.types.MutableType; import com.linkedin.datahub.graphql.types.SearchableEntityType; import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetMapper; import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetUpdateInputMapper; @@ -40,7 +41,9 @@ import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; + import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -56,7 +59,7 @@ public class DatasetType implements SearchableEntityType, BrowsableEntityType, - MutableType { + BatchMutableType { private static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( DATASET_KEY_ASPECT_NAME, @@ -76,7 +79,8 @@ public class DatasetType implements SearchableEntityType, Brows CONTAINER_ASPECT_NAME, DOMAINS_ASPECT_NAME, SCHEMA_METADATA_ASPECT_NAME, - DATA_PLATFORM_INSTANCE_ASPECT_NAME + DATA_PLATFORM_INSTANCE_ASPECT_NAME, + SIBLINGS_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); @@ -98,6 +102,11 @@ public Class inputClass() { return DatasetUpdateInput.class; } + @Override + public Class batchInputClass() { + return BatchDatasetUpdateInput[].class; + } + @Override public EntityType type() { return EntityType.DATASET; @@ -183,6 +192,30 @@ public List browsePaths(@Nonnull String urn, @Nonnull final QueryCon return BrowsePathsMapper.map(result); } + @Override + public List batchUpdate(@Nonnull BatchDatasetUpdateInput[] input, @Nonnull QueryContext context) throws Exception { + final Urn actor = Urn.createFromString(context.getAuthentication().getActor().toUrnStr()); + + final Collection proposals = Arrays.stream(input).map(updateInput -> { + if (isAuthorized(updateInput.getUrn(), updateInput.getUpdate(), context)) { + Collection datasetProposals = DatasetUpdateInputMapper.map(updateInput.getUpdate(), actor); + datasetProposals.forEach(proposal -> proposal.setEntityUrn(UrnUtils.getUrn(updateInput.getUrn()))); + return datasetProposals; + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }).flatMap(Collection::stream).collect(Collectors.toList()); + + final List urns = Arrays.stream(input).map(BatchDatasetUpdateInput::getUrn).collect(Collectors.toList()); + + try { + _entityClient.batchIngestProposals(proposals, context.getAuthentication()); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to write entity with urn %s", urns), e); + } + + return batchLoad(urns, context).stream().map(DataFetcherResult::getData).collect(Collectors.toList()); + } + @Override public Dataset update(@Nonnull String urn, @Nonnull DatasetUpdateInput input, @Nonnull QueryContext context) throws Exception { if (isAuthorized(urn, input, context)) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java index a8f61232a9aecc..d18c58a9e31663 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/VersionedDatasetType.java @@ -44,7 +44,8 @@ public class VersionedDatasetType implements com.linkedin.datahub.graphql.types. STATUS_ASPECT_NAME, CONTAINER_ASPECT_NAME, DOMAINS_ASPECT_NAME, - SCHEMA_METADATA_ASPECT_NAME + SCHEMA_METADATA_ASPECT_NAME, + SIBLINGS_ASPECT_NAME ); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 6a242c0f9426b0..6b300a9e2bb8b8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -6,22 +6,25 @@ import com.linkedin.common.GlossaryTerms; import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; +import com.linkedin.common.Siblings; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.Container; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.DatasetEditableProperties; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; +import com.linkedin.datahub.graphql.types.common.mappers.SiblingsMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -58,36 +61,39 @@ public static Dataset map(@Nonnull final EntityResponse dataset) { @Override public Dataset apply(@Nonnull final EntityResponse entityResponse) { Dataset result = new Dataset(); + Urn entityUrn = entityResponse.getUrn(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.DATASET); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(DATASET_KEY_ASPECT_NAME, this::mapDatasetKey); - mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, this::mapDatasetProperties); + mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, (entity, dataMap) -> this.mapDatasetProperties(entity, dataMap, entityUrn)); mappingHelper.mapToResult(DATASET_DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> dataset.setDeprecation(DatasetDeprecationMapper.map(new DatasetDeprecation(dataMap)))); mappingHelper.mapToResult(SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> - dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap)))); + dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap), entityUrn))); mappingHelper.mapToResult(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, this::mapEditableDatasetProperties); mappingHelper.mapToResult(VIEW_PROPERTIES_ASPECT_NAME, this::mapViewProperties); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataset, dataMap) -> dataset.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (dataset, dataMap) -> - dataset.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + dataset.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (dataset, dataMap) -> dataset.setStatus(StatusMapper.map(new Status(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (dataset, dataMap) -> this.mapGlobalTags(dataset, dataMap, entityUrn)); mappingHelper.mapToResult(EDITABLE_SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> - dataset.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(new EditableSchemaMetadata(dataMap)))); + dataset.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(new EditableSchemaMetadata(dataMap), entityUrn))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (dataset, dataMap) -> - dataset.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + dataset.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> dataset.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> dataset.setDataPlatformInstance(DataPlatformInstanceAspectMapper.map(new DataPlatformInstance(dataMap)))); + mappingHelper.mapToResult(SIBLINGS_ASPECT_NAME, (dataset, dataMap) -> + dataset.setSiblings(SiblingsMapper.map(new Siblings(dataMap)))); return mappingHelper.getResult(); } @@ -101,7 +107,7 @@ private void mapDatasetKey(@Nonnull Dataset dataset, @Nonnull DataMap dataMap) { .setUrn(gmsKey.getPlatform().toString()).build()); } - private void mapDatasetProperties(@Nonnull Dataset dataset, @Nonnull DataMap dataMap) { + private void mapDatasetProperties(@Nonnull Dataset dataset, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { final DatasetProperties gmsProperties = new DatasetProperties(dataMap); final com.linkedin.datahub.graphql.generated.DatasetProperties properties = new com.linkedin.datahub.graphql.generated.DatasetProperties(); @@ -111,7 +117,7 @@ private void mapDatasetProperties(@Nonnull Dataset dataset, @Nonnull DataMap dat if (gmsProperties.getExternalUrl() != null) { properties.setExternalUrl(gmsProperties.getExternalUrl().toString()); } - properties.setCustomProperties(StringMapMapper.map(gmsProperties.getCustomProperties())); + properties.setCustomProperties(CustomPropertiesMapper.map(gmsProperties.getCustomProperties(), entityUrn)); if (gmsProperties.getName() != null) { properties.setName(gmsProperties.getName()); } else { @@ -142,8 +148,8 @@ private void mapViewProperties(@Nonnull Dataset dataset, @Nonnull DataMap dataMa dataset.setViewProperties(graphqlProperties); } - private void mapGlobalTags(@Nonnull Dataset dataset, @Nonnull DataMap dataMap) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap)); + private void mapGlobalTags(@Nonnull Dataset dataset, @Nonnull DataMap dataMap, @Nonnull final Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); dataset.setGlobalTags(globalTags); dataset.setTags(globalTags); } @@ -159,11 +165,6 @@ private void mapContainers(@Nonnull Dataset dataset, @Nonnull DataMap dataMap) { private void mapDomains(@Nonnull Dataset dataset, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); - // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - dataset.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + dataset.setDomain(DomainAssociationMapper.map(domains, dataset.getUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java index e78295dc375476..922574d5051d30 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaFieldInfoMapper.java @@ -1,23 +1,28 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.schema.EditableSchemaFieldInfo; import javax.annotation.Nonnull; -public class EditableSchemaFieldInfoMapper implements ModelMapper { +public class EditableSchemaFieldInfoMapper { public static final EditableSchemaFieldInfoMapper INSTANCE = new EditableSchemaFieldInfoMapper(); - public static com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo map(@Nonnull final EditableSchemaFieldInfo fieldInfo) { - return INSTANCE.apply(fieldInfo); + public static com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo map( + @Nonnull final EditableSchemaFieldInfo fieldInfo, + @Nonnull final Urn entityUrn + ) { + return INSTANCE.apply(fieldInfo, entityUrn); } - @Override - public com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo apply(@Nonnull final EditableSchemaFieldInfo input) { + public com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo apply( + @Nonnull final EditableSchemaFieldInfo input, + @Nonnull final Urn entityUrn + ) { final com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo result = new com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo(); if (input.hasDescription()) { result.setDescription((input.getDescription())); @@ -26,11 +31,11 @@ public com.linkedin.datahub.graphql.generated.EditableSchemaFieldInfo apply(@Non result.setFieldPath((input.getFieldPath())); } if (input.hasGlobalTags()) { - result.setGlobalTags(GlobalTagsMapper.map(input.getGlobalTags())); - result.setTags(GlobalTagsMapper.map(input.getGlobalTags())); + result.setGlobalTags(GlobalTagsMapper.map(input.getGlobalTags(), entityUrn)); + result.setTags(GlobalTagsMapper.map(input.getGlobalTags(), entityUrn)); } if (input.hasGlossaryTerms()) { - result.setGlossaryTerms(GlossaryTermsMapper.map(input.getGlossaryTerms())); + result.setGlossaryTerms(GlossaryTermsMapper.map(input.getGlossaryTerms(), entityUrn)); } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java index 84803ed40a9220..376558d2fd18cb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/EditableSchemaMetadataMapper.java @@ -1,23 +1,27 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.schema.EditableSchemaMetadata; +import com.linkedin.common.urn.Urn; import javax.annotation.Nonnull; import java.util.stream.Collectors; -public class EditableSchemaMetadataMapper implements ModelMapper { +public class EditableSchemaMetadataMapper { public static final EditableSchemaMetadataMapper INSTANCE = new EditableSchemaMetadataMapper(); - public static com.linkedin.datahub.graphql.generated.EditableSchemaMetadata map(@Nonnull final EditableSchemaMetadata metadata) { - return INSTANCE.apply(metadata); + public static com.linkedin.datahub.graphql.generated.EditableSchemaMetadata map( + @Nonnull final EditableSchemaMetadata metadata, + @Nonnull final Urn entityUrn + ) { + return INSTANCE.apply(metadata, entityUrn); } - @Override - public com.linkedin.datahub.graphql.generated.EditableSchemaMetadata apply(@Nonnull final EditableSchemaMetadata input) { + public com.linkedin.datahub.graphql.generated.EditableSchemaMetadata apply(@Nonnull final EditableSchemaMetadata input, @Nonnull final Urn entityUrn) { final com.linkedin.datahub.graphql.generated.EditableSchemaMetadata result = new com.linkedin.datahub.graphql.generated.EditableSchemaMetadata(); - result.setEditableSchemaFieldInfo(input.getEditableSchemaFieldInfo().stream().map(EditableSchemaFieldInfoMapper::map).collect(Collectors.toList())); + result.setEditableSchemaFieldInfo(input.getEditableSchemaFieldInfo().stream().map(schemaField -> + EditableSchemaFieldInfoMapper.map(schemaField, entityUrn) + ).collect(Collectors.toList())); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java index e9cb524956bcc8..c5308f453e147a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java @@ -1,23 +1,22 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.SchemaField; import com.linkedin.datahub.graphql.generated.SchemaFieldDataType; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import javax.annotation.Nonnull; -public class SchemaFieldMapper implements ModelMapper { +public class SchemaFieldMapper { public static final SchemaFieldMapper INSTANCE = new SchemaFieldMapper(); - public static SchemaField map(@Nonnull final com.linkedin.schema.SchemaField metadata) { - return INSTANCE.apply(metadata); + public static SchemaField map(@Nonnull final com.linkedin.schema.SchemaField metadata, @Nonnull Urn entityUrn) { + return INSTANCE.apply(metadata, entityUrn); } - @Override - public SchemaField apply(@Nonnull final com.linkedin.schema.SchemaField input) { + public SchemaField apply(@Nonnull final com.linkedin.schema.SchemaField input, @Nonnull Urn entityUrn) { final SchemaField result = new SchemaField(); result.setDescription(input.getDescription()); result.setFieldPath(input.getFieldPath()); @@ -27,11 +26,11 @@ public SchemaField apply(@Nonnull final com.linkedin.schema.SchemaField input) { result.setNativeDataType(input.getNativeDataType()); result.setType(mapSchemaFieldDataType(input.getType())); if (input.hasGlobalTags()) { - result.setGlobalTags(GlobalTagsMapper.map(input.getGlobalTags())); - result.setTags(GlobalTagsMapper.map(input.getGlobalTags())); + result.setGlobalTags(GlobalTagsMapper.map(input.getGlobalTags(), entityUrn)); + result.setTags(GlobalTagsMapper.map(input.getGlobalTags(), entityUrn)); } if (input.hasGlossaryTerms()) { - result.setGlossaryTerms(GlossaryTermsMapper.map(input.getGlossaryTerms())); + result.setGlossaryTerms(GlossaryTermsMapper.map(input.getGlossaryTerms(), entityUrn)); } result.setIsPartOfKey(input.isIsPartOfKey()); return result; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java index 7e068d5d414e0e..eb793cc17efb6b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java @@ -1,33 +1,41 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.Schema; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.SchemaMetadata; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.stream.Collectors; -public class SchemaMapper implements ModelMapper { +public class SchemaMapper { public static final SchemaMapper INSTANCE = new SchemaMapper(); - public static Schema map(@Nonnull final SchemaMetadata metadata) { - return INSTANCE.apply(metadata); + public static Schema map(@Nonnull final SchemaMetadata metadata, @Nonnull final Urn entityUrn) { + return INSTANCE.apply(metadata, null, entityUrn); } - @Override - public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { + public static Schema map(@Nonnull final SchemaMetadata metadata, @Nullable final SystemMetadata systemMetadata, @Nonnull final Urn entityUrn) { + return INSTANCE.apply(metadata, systemMetadata, entityUrn); + } + + public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input, @Nullable final SystemMetadata systemMetadata, @Nonnull final Urn entityUrn) { final Schema result = new Schema(); if (input.getDataset() != null) { result.setDatasetUrn(input.getDataset().toString()); } + if (systemMetadata != null) { + result.setLastObserved(systemMetadata.getLastObserved()); + } result.setName(input.getSchemaName()); result.setPlatformUrn(input.getPlatform().toString()); result.setVersion(input.getVersion()); result.setCluster(input.getCluster()); result.setHash(input.getHash()); result.setPrimaryKeys(input.getPrimaryKeys()); - result.setFields(input.getFields().stream().map(SchemaFieldMapper::map).collect(Collectors.toList())); + result.setFields(input.getFields().stream().map(field -> SchemaFieldMapper.map(field, entityUrn)).collect(Collectors.toList())); result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); if (input.getForeignKeys() != null) { result.setForeignKeys(input.getForeignKeys().stream() diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java index c89de8249a10fd..00cb91bed8abb2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java @@ -1,22 +1,27 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.common.urn.Urn; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.schema.SchemaMetadata; import java.util.stream.Collectors; import javax.annotation.Nonnull; -public class SchemaMetadataMapper implements ModelMapper { +public class SchemaMetadataMapper { public static final SchemaMetadataMapper INSTANCE = new SchemaMetadataMapper(); - public static com.linkedin.datahub.graphql.generated.SchemaMetadata map(@Nonnull final EnvelopedAspect aspect) { - return INSTANCE.apply(aspect); + public static com.linkedin.datahub.graphql.generated.SchemaMetadata map( + @Nonnull final EnvelopedAspect aspect, + @Nonnull final Urn entityUrn + ) { + return INSTANCE.apply(aspect, entityUrn); } - @Override - public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(@Nonnull final EnvelopedAspect aspect) { + public com.linkedin.datahub.graphql.generated.SchemaMetadata apply( + @Nonnull final EnvelopedAspect aspect, + @Nonnull final Urn entityUrn + ) { final SchemaMetadata input = new SchemaMetadata(aspect.getValue().data()); final com.linkedin.datahub.graphql.generated.SchemaMetadata result = new com.linkedin.datahub.graphql.generated.SchemaMetadata(); @@ -30,7 +35,7 @@ public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(@Nonnull fina result.setCluster(input.getCluster()); result.setHash(input.getHash()); result.setPrimaryKeys(input.getPrimaryKeys()); - result.setFields(input.getFields().stream().map(SchemaFieldMapper::map).collect(Collectors.toList())); + result.setFields(input.getFields().stream().map(field -> SchemaFieldMapper.map(field, entityUrn)).collect(Collectors.toList())); result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); result.setAspectVersion(aspect.getVersion()); if (input.hasForeignKeys()) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java index 5f9d4944e5ae27..5d9d40970ac434 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/VersionedDatasetMapper.java @@ -6,11 +6,11 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.Container; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.DatasetEditableProperties; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.generated.VersionedDataset; @@ -18,8 +18,9 @@ import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -31,6 +32,7 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.metadata.key.DatasetKey; +import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.EditableSchemaMetadata; import com.linkedin.schema.SchemaMetadata; import javax.annotation.Nonnull; @@ -56,30 +58,33 @@ public static VersionedDataset map(@Nonnull final EntityResponse dataset) { @Override public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) { VersionedDataset result = new VersionedDataset(); + Urn entityUrn = entityResponse.getUrn(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.DATASET); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + SystemMetadata schemaSystemMetadata = getSystemMetadata(aspectMap, SCHEMA_METADATA_ASPECT_NAME); + mappingHelper.mapToResult(DATASET_KEY_ASPECT_NAME, this::mapDatasetKey); - mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, this::mapDatasetProperties); + mappingHelper.mapToResult(DATASET_PROPERTIES_ASPECT_NAME, (entity, dataMap) -> this.mapDatasetProperties(entity, dataMap, entityUrn)); mappingHelper.mapToResult(DATASET_DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> dataset.setDeprecation(DatasetDeprecationMapper.map(new DatasetDeprecation(dataMap)))); mappingHelper.mapToResult(SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> - dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap)))); + dataset.setSchema(SchemaMapper.map(new SchemaMetadata(dataMap), schemaSystemMetadata, entityUrn))); mappingHelper.mapToResult(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, this::mapEditableDatasetProperties); mappingHelper.mapToResult(VIEW_PROPERTIES_ASPECT_NAME, this::mapViewProperties); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataset, dataMap) -> dataset.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (dataset, dataMap) -> - dataset.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + dataset.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (dataset, dataMap) -> dataset.setStatus(StatusMapper.map(new Status(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (dataset, dataMap) -> this.mapGlobalTags(dataset, dataMap, entityUrn)); mappingHelper.mapToResult(EDITABLE_SCHEMA_METADATA_ASPECT_NAME, (dataset, dataMap) -> - dataset.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(new EditableSchemaMetadata(dataMap)))); + dataset.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(new EditableSchemaMetadata(dataMap), entityUrn))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (dataset, dataMap) -> - dataset.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + dataset.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (dataset, dataMap) -> @@ -88,6 +93,13 @@ public VersionedDataset apply(@Nonnull final EntityResponse entityResponse) { return mappingHelper.getResult(); } + private SystemMetadata getSystemMetadata(EnvelopedAspectMap aspectMap, String aspectName) { + if (aspectMap.containsKey(aspectName) && aspectMap.get(aspectName).hasSystemMetadata()) { + return aspectMap.get(aspectName).getSystemMetadata(); + } + return null; + } + private void mapDatasetKey(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { final DatasetKey gmsKey = new DatasetKey(dataMap); dataset.setName(gmsKey.getName()); @@ -97,7 +109,7 @@ private void mapDatasetKey(@Nonnull VersionedDataset dataset, @Nonnull DataMap d .setUrn(gmsKey.getPlatform().toString()).build()); } - private void mapDatasetProperties(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { + private void mapDatasetProperties(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap, Urn entityUrn) { final DatasetProperties gmsProperties = new DatasetProperties(dataMap); final com.linkedin.datahub.graphql.generated.DatasetProperties properties = new com.linkedin.datahub.graphql.generated.DatasetProperties(); @@ -106,7 +118,7 @@ private void mapDatasetProperties(@Nonnull VersionedDataset dataset, @Nonnull Da if (gmsProperties.getExternalUrl() != null) { properties.setExternalUrl(gmsProperties.getExternalUrl().toString()); } - properties.setCustomProperties(StringMapMapper.map(gmsProperties.getCustomProperties())); + properties.setCustomProperties(CustomPropertiesMapper.map(gmsProperties.getCustomProperties(), entityUrn)); if (gmsProperties.getName() != null) { properties.setName(gmsProperties.getName()); } else { @@ -133,8 +145,8 @@ private void mapViewProperties(@Nonnull VersionedDataset dataset, @Nonnull DataM dataset.setViewProperties(graphqlProperties); } - private void mapGlobalTags(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { - com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap)); + private void mapGlobalTags(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); dataset.setTags(globalTags); } @@ -150,10 +162,6 @@ private void mapContainers(@Nonnull VersionedDataset dataset, @Nonnull DataMap d private void mapDomains(@Nonnull VersionedDataset dataset, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - dataset.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + dataset.setDomain(DomainAssociationMapper.map(domains, dataset.getUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainAssociationMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainAssociationMapper.java new file mode 100644 index 00000000000000..df8de87ff69ff1 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainAssociationMapper.java @@ -0,0 +1,36 @@ +package com.linkedin.datahub.graphql.types.domain; + +import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.DomainAssociation; +import com.linkedin.datahub.graphql.generated.EntityType; +import javax.annotation.Nonnull; + + +/** + * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. + * + * To be replaced by auto-generated mappers implementations + */ +public class DomainAssociationMapper { + + public static final DomainAssociationMapper INSTANCE = new DomainAssociationMapper(); + + public static DomainAssociation map( + @Nonnull final com.linkedin.domain.Domains domains, + @Nonnull final String entityUrn + ) { + return INSTANCE.apply(domains, entityUrn); + } + + public DomainAssociation apply(@Nonnull final com.linkedin.domain.Domains domains, @Nonnull final String entityUrn) { + if (domains.getDomains().size() > 0) { + DomainAssociation association = new DomainAssociation(); + association.setDomain(Domain.builder() + .setType(EntityType.DOMAIN) + .setUrn(domains.getDomains().get(0).toString()).build()); + association.setAssociatedUrn(entityUrn); + return association; + } + return null; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java index 9f107aeb0dece7..98919ff1f44303 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java @@ -40,7 +40,7 @@ public static Domain map(final EntityResponse entityResponse) { final EnvelopedAspect envelopedOwnership = aspects.get(Constants.OWNERSHIP_ASPECT_NAME); if (envelopedOwnership != null) { - result.setOwnership(OwnershipMapper.map(new Ownership(envelopedOwnership.getValue().data()))); + result.setOwnership(OwnershipMapper.map(new Ownership(envelopedOwnership.getValue().data()), entityUrn)); } final EnvelopedAspect envelopedInstitutionalMemory = aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME); @@ -59,4 +59,4 @@ private static com.linkedin.datahub.graphql.generated.DomainProperties mapDomain } private DomainMapper() { } -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java index e0d4db0ac5cfa3..6a1d849dd23bf5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.types.glossary.mappers; import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.GlossaryNode; @@ -30,6 +31,7 @@ public GlossaryNode apply(@Nonnull final EntityResponse entityResponse) { GlossaryNode result = new GlossaryNode(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.GLOSSARY_NODE); + Urn entityUrn = entityResponse.getUrn(); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); @@ -37,7 +39,7 @@ public GlossaryNode apply(@Nonnull final EntityResponse entityResponse) { glossaryNode.setProperties(mapGlossaryNodeProperties(dataMap))); mappingHelper.mapToResult(GLOSSARY_NODE_KEY_ASPECT_NAME, this::mapGlossaryNodeKey); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (glossaryNode, dataMap) -> - glossaryNode.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + glossaryNode.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); return mappingHelper.getResult(); } @@ -54,6 +56,11 @@ private GlossaryNodeProperties mapGlossaryNodeProperties(@Nonnull DataMap dataMa private void mapGlossaryNodeKey(@Nonnull GlossaryNode glossaryNode, @Nonnull DataMap dataMap) { GlossaryNodeKey glossaryNodeKey = new GlossaryNodeKey(dataMap); + + if (glossaryNode.getProperties() == null) { + glossaryNode.setProperties(new GlossaryNodeProperties()); + } + if (glossaryNode.getProperties().getName() == null) { glossaryNode.getProperties().setName(glossaryNodeKey.getName()); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermInfoMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermInfoMapper.java index 9885cce7176a3c..2f99700bc30a14 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermInfoMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermInfoMapper.java @@ -1,26 +1,25 @@ package com.linkedin.datahub.graphql.types.glossary.mappers; +import com.linkedin.common.urn.Urn; import javax.annotation.Nonnull; import com.linkedin.datahub.graphql.generated.GlossaryTermInfo; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; /** * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. * * To be replaced by auto-generated mappers implementations */ -public class GlossaryTermInfoMapper implements ModelMapper { +public class GlossaryTermInfoMapper { public static final GlossaryTermInfoMapper INSTANCE = new GlossaryTermInfoMapper(); - public static GlossaryTermInfo map(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo) { - return INSTANCE.apply(glossaryTermInfo); + public static GlossaryTermInfo map(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo, Urn entityUrn) { + return INSTANCE.apply(glossaryTermInfo, entityUrn); } - @Override - public GlossaryTermInfo apply(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo) { + public GlossaryTermInfo apply(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo, Urn entityUrn) { com.linkedin.datahub.graphql.generated.GlossaryTermInfo glossaryTermInfoResult = new com.linkedin.datahub.graphql.generated.GlossaryTermInfo(); glossaryTermInfoResult.setDefinition(glossaryTermInfo.getDefinition()); glossaryTermInfoResult.setDescription(glossaryTermInfo.getDefinition()); @@ -35,7 +34,7 @@ public GlossaryTermInfo apply(@Nonnull final com.linkedin.glossary.GlossaryTermI glossaryTermInfoResult.setSourceUrl(glossaryTermInfo.getSourceUrl().toString()); } if (glossaryTermInfo.hasCustomProperties()) { - glossaryTermInfoResult.setCustomProperties(StringMapMapper.map(glossaryTermInfo.getCustomProperties())); + glossaryTermInfoResult.setCustomProperties(CustomPropertiesMapper.map(glossaryTermInfo.getCustomProperties(), entityUrn)); } return glossaryTermInfoResult; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java index e7406673c0b1d2..99cbb7d04a9745 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java @@ -3,6 +3,7 @@ import com.linkedin.common.Deprecation; import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.generated.EntityType; @@ -38,6 +39,8 @@ public static GlossaryTerm map(@Nonnull final EntityResponse entityResponse) { @Override public GlossaryTerm apply(@Nonnull final EntityResponse entityResponse) { GlossaryTerm result = new GlossaryTerm(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.GLOSSARY_TERM); final String legacyName = GlossaryTermUtils.getGlossaryTermName(entityResponse.getUrn().getId()); @@ -46,11 +49,11 @@ public GlossaryTerm apply(@Nonnull final EntityResponse entityResponse) { MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(GLOSSARY_TERM_KEY_ASPECT_NAME, this::mapGlossaryTermKey); mappingHelper.mapToResult(GLOSSARY_TERM_INFO_ASPECT_NAME, (glossaryTerm, dataMap) -> - glossaryTerm.setGlossaryTermInfo(GlossaryTermInfoMapper.map(new GlossaryTermInfo(dataMap)))); + glossaryTerm.setGlossaryTermInfo(GlossaryTermInfoMapper.map(new GlossaryTermInfo(dataMap), entityUrn))); mappingHelper.mapToResult(GLOSSARY_TERM_INFO_ASPECT_NAME, (glossaryTerm, dataMap) -> - glossaryTerm.setProperties(GlossaryTermPropertiesMapper.map(new GlossaryTermInfo(dataMap)))); + glossaryTerm.setProperties(GlossaryTermPropertiesMapper.map(new GlossaryTermInfo(dataMap), entityUrn))); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (glossaryTerm, dataMap) -> - glossaryTerm.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + glossaryTerm.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (glossaryTerm, dataMap) -> glossaryTerm.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (dataset, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermPropertiesMapper.java index d56f63ac8854cf..6b358331833937 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermPropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermPropertiesMapper.java @@ -1,26 +1,25 @@ package com.linkedin.datahub.graphql.types.glossary.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.GlossaryTermProperties; import javax.annotation.Nonnull; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; /** * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. * * To be replaced by auto-generated mappers implementations */ -public class GlossaryTermPropertiesMapper implements ModelMapper { +public class GlossaryTermPropertiesMapper { public static final GlossaryTermPropertiesMapper INSTANCE = new GlossaryTermPropertiesMapper(); - public static GlossaryTermProperties map(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo) { - return INSTANCE.apply(glossaryTermInfo); + public static GlossaryTermProperties map(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo, Urn entityUrn) { + return INSTANCE.apply(glossaryTermInfo, entityUrn); } - @Override - public GlossaryTermProperties apply(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo) { + public GlossaryTermProperties apply(@Nonnull final com.linkedin.glossary.GlossaryTermInfo glossaryTermInfo, Urn entityUrn) { com.linkedin.datahub.graphql.generated.GlossaryTermProperties result = new com.linkedin.datahub.graphql.generated.GlossaryTermProperties(); result.setDefinition(glossaryTermInfo.getDefinition()); result.setDescription(glossaryTermInfo.getDefinition()); @@ -35,7 +34,7 @@ public GlossaryTermProperties apply(@Nonnull final com.linkedin.glossary.Glossar result.setSourceUrl(glossaryTermInfo.getSourceUrl().toString()); } if (glossaryTermInfo.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(glossaryTermInfo.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(glossaryTermInfo.getCustomProperties(), entityUrn)); } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java index 9e88f12e355b13..a64b0f7dc64fbe 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.glossary.mappers; +import com.linkedin.common.urn.Urn; import javax.annotation.Nonnull; import java.util.stream.Collectors; @@ -7,7 +8,6 @@ import com.linkedin.datahub.graphql.generated.GlossaryTerms; import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.datahub.graphql.generated.GlossaryTerm; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.glossary.GlossaryTermUtils; /** @@ -15,29 +15,37 @@ * * To be replaced by auto-generated mappers implementations */ -public class GlossaryTermsMapper implements ModelMapper { +public class GlossaryTermsMapper { public static final GlossaryTermsMapper INSTANCE = new GlossaryTermsMapper(); - public static GlossaryTerms map(@Nonnull final com.linkedin.common.GlossaryTerms glossaryTerms) { - return INSTANCE.apply(glossaryTerms); + public static GlossaryTerms map( + @Nonnull final com.linkedin.common.GlossaryTerms glossaryTerms, + @Nonnull final Urn entityUrn + ) { + return INSTANCE.apply(glossaryTerms, entityUrn); } - @Override - public GlossaryTerms apply(@Nonnull final com.linkedin.common.GlossaryTerms glossaryTerms) { + public GlossaryTerms apply(@Nonnull final com.linkedin.common.GlossaryTerms glossaryTerms, @Nonnull final Urn entityUrn) { com.linkedin.datahub.graphql.generated.GlossaryTerms result = new com.linkedin.datahub.graphql.generated.GlossaryTerms(); - result.setTerms(glossaryTerms.getTerms().stream().map(this::mapGlossaryTermAssociation).collect(Collectors.toList())); + result.setTerms(glossaryTerms.getTerms().stream().map( + association -> this.mapGlossaryTermAssociation(association, entityUrn) + ).collect(Collectors.toList())); return result; } - private com.linkedin.datahub.graphql.generated.GlossaryTermAssociation mapGlossaryTermAssociation(@Nonnull final GlossaryTermAssociation input) { + private com.linkedin.datahub.graphql.generated.GlossaryTermAssociation mapGlossaryTermAssociation( + @Nonnull final GlossaryTermAssociation input, + @Nonnull final Urn entityUrn + ) { final com.linkedin.datahub.graphql.generated.GlossaryTermAssociation result = new com.linkedin.datahub.graphql.generated.GlossaryTermAssociation(); final GlossaryTerm resultGlossaryTerm = new GlossaryTerm(); resultGlossaryTerm.setType(EntityType.GLOSSARY_TERM); resultGlossaryTerm.setUrn(input.getUrn().toString()); resultGlossaryTerm.setName(GlossaryTermUtils.getGlossaryTermName(input.getUrn().getNameEntity())); result.setTerm(resultGlossaryTerm); + result.setAssociatedUrn(entityUrn.toString()); return result; } -} \ No newline at end of file +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java index db2ba676234656..fdb151af321496 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java @@ -9,9 +9,9 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.MLFeature; import com.linkedin.datahub.graphql.generated.MLFeatureDataType; @@ -22,6 +22,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -51,13 +52,15 @@ public static MLFeature map(@Nonnull final EntityResponse entityResponse) { @Override public MLFeature apply(@Nonnull final EntityResponse entityResponse) { final MLFeature result = new MLFeature(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.MLFEATURE); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(ML_FEATURE_KEY_ASPECT_NAME, this::mapMLFeatureKey); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlFeature, dataMap) -> - mlFeature.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + mlFeature.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(ML_FEATURE_PROPERTIES_ASPECT_NAME, this::mapMLFeatureProperties); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (mlFeature, dataMap) -> mlFeature.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); @@ -66,9 +69,9 @@ public MLFeature apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (mlFeature, dataMap) -> mlFeature.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (entity, dataMap) -> this.mapGlobalTags(entity, dataMap, entityUrn)); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (entity, dataMap) -> - entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(ML_FEATURE_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> @@ -93,20 +96,16 @@ private void mapMLFeatureProperties(@Nonnull MLFeature mlFeature, @Nonnull DataM } } - private void mapGlobalTags(MLFeature entity, DataMap dataMap) { + private void mapGlobalTags(MLFeature entity, DataMap dataMap, Urn entityUrn) { GlobalTags globalTags = new GlobalTags(dataMap); - com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags); + com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags, entityUrn); entity.setTags(graphQlGlobalTags); } private void mapDomains(@Nonnull MLFeature entity, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - entity.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + entity.setDomain(DomainAssociationMapper.map(domains, entity.getUrn())); } private void mapEditableProperties(MLFeature entity, DataMap dataMap) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java index f700e44b4f564e..a881cb9313274d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java @@ -8,10 +8,10 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.MLFeatureTable; import com.linkedin.datahub.graphql.generated.MLFeatureTableEditableProperties; @@ -21,6 +21,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -50,14 +51,16 @@ public static MLFeatureTable map(@Nonnull final EntityResponse entityResponse) { @Override public MLFeatureTable apply(@Nonnull final EntityResponse entityResponse) { final MLFeatureTable result = new MLFeatureTable(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.MLFEATURE_TABLE); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlFeatureTable, dataMap) -> - mlFeatureTable.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + mlFeatureTable.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(ML_FEATURE_TABLE_KEY_ASPECT_NAME, this::mapMLFeatureTableKey); - mappingHelper.mapToResult(ML_FEATURE_TABLE_PROPERTIES_ASPECT_NAME, this::mapMLFeatureTableProperties); + mappingHelper.mapToResult(ML_FEATURE_TABLE_PROPERTIES_ASPECT_NAME, (entity, dataMap) -> this.mapMLFeatureTableProperties(entity, dataMap, entityUrn)); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (mlFeatureTable, dataMap) -> mlFeatureTable.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (mlFeatureTable, dataMap) -> @@ -65,9 +68,9 @@ public MLFeatureTable apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (mlFeatureTable, dataMap) -> mlFeatureTable.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (entity, dataMap) -> this.mapGlobalTags(entity, dataMap, entityUrn)); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (entity, dataMap) -> - entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(ML_FEATURE_TABLE_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> @@ -84,27 +87,23 @@ private void mapMLFeatureTableKey(@Nonnull MLFeatureTable mlFeatureTable, @Nonnu mlFeatureTable.setPlatform(partialPlatform); } - private void mapMLFeatureTableProperties(@Nonnull MLFeatureTable mlFeatureTable, @Nonnull DataMap dataMap) { + private void mapMLFeatureTableProperties(@Nonnull MLFeatureTable mlFeatureTable, @Nonnull DataMap dataMap, Urn entityUrn) { MLFeatureTableProperties featureTableProperties = new MLFeatureTableProperties(dataMap); - mlFeatureTable.setFeatureTableProperties(MLFeatureTablePropertiesMapper.map(featureTableProperties)); - mlFeatureTable.setProperties(MLFeatureTablePropertiesMapper.map(featureTableProperties)); + mlFeatureTable.setFeatureTableProperties(MLFeatureTablePropertiesMapper.map(featureTableProperties, entityUrn)); + mlFeatureTable.setProperties(MLFeatureTablePropertiesMapper.map(featureTableProperties, entityUrn)); mlFeatureTable.setDescription(featureTableProperties.getDescription()); } - private void mapGlobalTags(MLFeatureTable entity, DataMap dataMap) { + private void mapGlobalTags(MLFeatureTable entity, DataMap dataMap, Urn entityUrn) { GlobalTags globalTags = new GlobalTags(dataMap); - com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags); + com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags, entityUrn); entity.setTags(graphQlGlobalTags); } private void mapDomains(@Nonnull MLFeatureTable entity, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - entity.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + entity.setDomain(DomainAssociationMapper.map(domains, entity.getUrn())); } private void mapEditableProperties(MLFeatureTable entity, DataMap dataMap) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java index 578b7d0731f4a4..13e3c795997250 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTablePropertiesMapper.java @@ -1,24 +1,23 @@ package com.linkedin.datahub.graphql.types.mlmodel.mappers; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.MLFeature; import com.linkedin.datahub.graphql.generated.MLFeatureTableProperties; import com.linkedin.datahub.graphql.generated.MLPrimaryKey; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import lombok.NonNull; import java.util.stream.Collectors; -public class MLFeatureTablePropertiesMapper implements ModelMapper { +public class MLFeatureTablePropertiesMapper { public static final MLFeatureTablePropertiesMapper INSTANCE = new MLFeatureTablePropertiesMapper(); - public static MLFeatureTableProperties map(@NonNull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties) { - return INSTANCE.apply(mlFeatureTableProperties); + public static MLFeatureTableProperties map(@NonNull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties, Urn entityUrn) { + return INSTANCE.apply(mlFeatureTableProperties, entityUrn); } - @Override - public MLFeatureTableProperties apply(@NonNull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties) { + public MLFeatureTableProperties apply(@NonNull final com.linkedin.ml.metadata.MLFeatureTableProperties mlFeatureTableProperties, Urn entityUrn) { final MLFeatureTableProperties result = new MLFeatureTableProperties(); result.setDescription(mlFeatureTableProperties.getDescription()); @@ -43,7 +42,7 @@ public MLFeatureTableProperties apply(@NonNull final com.linkedin.ml.metadata.ML } if (mlFeatureTableProperties.hasCustomProperties()) { - result.setCustomProperties(StringMapMapper.map(mlFeatureTableProperties.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(mlFeatureTableProperties.getCustomProperties(), entityUrn)); } return result; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java index 36d2918bf4f041..a4ae4fc968707f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java @@ -6,10 +6,10 @@ import com.linkedin.common.GlossaryTerms; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.generated.MLModelGroup; @@ -19,6 +19,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -48,12 +49,14 @@ public static MLModelGroup map(@Nonnull final EntityResponse entityResponse) { @Override public MLModelGroup apply(@Nonnull final EntityResponse entityResponse) { final MLModelGroup result = new MLModelGroup(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.MLMODEL_GROUP); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlModelGroup, dataMap) -> - mlModelGroup.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + mlModelGroup.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(ML_MODEL_GROUP_KEY_ASPECT_NAME, this::mapToMLModelGroupKey); mappingHelper.mapToResult(ML_MODEL_GROUP_PROPERTIES_ASPECT_NAME, this::mapToMLModelGroupProperties); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (mlModelGroup, dataMap) -> @@ -61,9 +64,9 @@ public MLModelGroup apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (mlModelGroup, dataMap) -> mlModelGroup.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (entity, dataMap) -> this.mapGlobalTags(entity, dataMap, entityUrn)); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (entity, dataMap) -> - entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(ML_MODEL_GROUP_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> @@ -89,20 +92,16 @@ private void mapToMLModelGroupProperties(MLModelGroup mlModelGroup, DataMap data } } - private void mapGlobalTags(MLModelGroup entity, DataMap dataMap) { + private void mapGlobalTags(MLModelGroup entity, DataMap dataMap, Urn entityUrn) { GlobalTags globalTags = new GlobalTags(dataMap); - com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags); + com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags, entityUrn); entity.setTags(graphQlGlobalTags); } private void mapDomains(@Nonnull MLModelGroup entity, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - entity.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + entity.setDomain(DomainAssociationMapper.map(domains, entity.getUrn())); } private void mapEditableProperties(MLModelGroup entity, DataMap dataMap) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java index 5df8e86a310dd0..71a19cd69d8072 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java @@ -8,10 +8,10 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FabricType; import com.linkedin.datahub.graphql.generated.MLModel; @@ -23,6 +23,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -62,15 +63,17 @@ public static MLModel map(@Nonnull final EntityResponse entityResponse) { @Override public MLModel apply(@Nonnull final EntityResponse entityResponse) { final MLModel result = new MLModel(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.MLMODEL); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(ML_MODEL_KEY_ASPECT_NAME, this::mapMLModelKey); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlModel, dataMap) -> - mlModel.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); - mappingHelper.mapToResult(ML_MODEL_PROPERTIES_ASPECT_NAME, this::mapMLModelProperties); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mlModel.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); + mappingHelper.mapToResult(ML_MODEL_PROPERTIES_ASPECT_NAME, (entity, dataMap) -> this.mapMLModelProperties(entity, dataMap, entityUrn)); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (mlModel, dataMap) -> this.mapGlobalTags(mlModel, dataMap, entityUrn)); mappingHelper.mapToResult(INTENDED_USE_ASPECT_NAME, (mlModel, dataMap) -> mlModel.setIntendedUse(IntendedUseMapper.map(new IntendedUse(dataMap)))); mappingHelper.mapToResult(ML_MODEL_FACTOR_PROMPTS_ASPECT_NAME, (mlModel, dataMap) -> @@ -101,7 +104,7 @@ public MLModel apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (mlModel, dataMap) -> mlModel.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (entity, dataMap) -> - entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(ML_MODEL_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> @@ -119,17 +122,17 @@ private void mapMLModelKey(MLModel mlModel, DataMap dataMap) { mlModel.setPlatform(partialPlatform); } - private void mapMLModelProperties(MLModel mlModel, DataMap dataMap) { + private void mapMLModelProperties(MLModel mlModel, DataMap dataMap, Urn entityUrn) { MLModelProperties modelProperties = new MLModelProperties(dataMap); - mlModel.setProperties(MLModelPropertiesMapper.map(modelProperties)); + mlModel.setProperties(MLModelPropertiesMapper.map(modelProperties, entityUrn)); if (modelProperties.getDescription() != null) { mlModel.setDescription(modelProperties.getDescription()); } } - private void mapGlobalTags(MLModel mlModel, DataMap dataMap) { + private void mapGlobalTags(MLModel mlModel, DataMap dataMap, Urn entityUrn) { GlobalTags globalTags = new GlobalTags(dataMap); - com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags); + com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags, entityUrn); mlModel.setGlobalTags(graphQlGlobalTags); mlModel.setTags(graphQlGlobalTags); } @@ -143,15 +146,10 @@ private void mapSourceCode(MLModel mlModel, DataMap dataMap) { mlModel.setSourceCode(graphQlSourceCode); } - private void mapDomains(@Nonnull MLModel entity, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - entity.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + entity.setDomain(DomainAssociationMapper.map(domains, entity.getUrn())); } private void mapEditableProperties(MLModel entity, DataMap dataMap) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java index e19bb9b2738602..950cfcc0689667 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelPropertiesMapper.java @@ -2,25 +2,23 @@ import com.linkedin.datahub.graphql.generated.MLModelGroup; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import java.util.stream.Collectors; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.MLModelProperties; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import lombok.NonNull; -public class MLModelPropertiesMapper implements ModelMapper { +public class MLModelPropertiesMapper { public static final MLModelPropertiesMapper INSTANCE = new MLModelPropertiesMapper(); - public static MLModelProperties map(@NonNull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties) { - return INSTANCE.apply(mlModelProperties); + public static MLModelProperties map(@NonNull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties, Urn entityUrn) { + return INSTANCE.apply(mlModelProperties, entityUrn); } - @Override - public MLModelProperties apply(@NonNull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties) { + public MLModelProperties apply(@NonNull final com.linkedin.ml.metadata.MLModelProperties mlModelProperties, Urn entityUrn) { final MLModelProperties result = new MLModelProperties(); result.setDate(mlModelProperties.getDate()); @@ -34,7 +32,7 @@ public MLModelProperties apply(@NonNull final com.linkedin.ml.metadata.MLModelPr param -> MLHyperParamMapper.map(param)).collect(Collectors.toList())); } - result.setCustomProperties(StringMapMapper.map(mlModelProperties.getCustomProperties())); + result.setCustomProperties(CustomPropertiesMapper.map(mlModelProperties.getCustomProperties(), entityUrn)); if (mlModelProperties.getTrainingMetrics() != null) { result.setTrainingMetrics(mlModelProperties.getTrainingMetrics().stream().map(metric -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java index e33921fce14ec6..b7d83a6111d9f4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java @@ -7,9 +7,9 @@ import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.MLFeatureDataType; import com.linkedin.datahub.graphql.generated.MLPrimaryKey; @@ -20,6 +20,7 @@ import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -49,12 +50,14 @@ public static MLPrimaryKey map(@Nonnull final EntityResponse entityResponse) { @Override public MLPrimaryKey apply(@Nonnull final EntityResponse entityResponse) { final MLPrimaryKey result = new MLPrimaryKey(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.MLPRIMARY_KEY); EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (mlPrimaryKey, dataMap) -> - mlPrimaryKey.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + mlPrimaryKey.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); mappingHelper.mapToResult(ML_PRIMARY_KEY_KEY_ASPECT_NAME, this::mapMLPrimaryKeyKey); mappingHelper.mapToResult(ML_PRIMARY_KEY_PROPERTIES_ASPECT_NAME, this::mapMLPrimaryKeyProperties); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (mlPrimaryKey, dataMap) -> @@ -64,9 +67,9 @@ public MLPrimaryKey apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(DEPRECATION_ASPECT_NAME, (mlPrimaryKey, dataMap) -> mlPrimaryKey.setDeprecation(DeprecationMapper.map(new Deprecation(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, this::mapGlobalTags); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (entity, dataMap) -> this.mapGlobalTags(entity, dataMap, entityUrn)); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (entity, dataMap) -> - entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + entity.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(ML_PRIMARY_KEY_EDITABLE_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataset, dataMap) -> @@ -90,20 +93,16 @@ private void mapMLPrimaryKeyProperties(MLPrimaryKey mlPrimaryKey, DataMap dataMa } } - private void mapGlobalTags(MLPrimaryKey entity, DataMap dataMap) { + private void mapGlobalTags(MLPrimaryKey entity, DataMap dataMap, Urn entityUrn) { GlobalTags globalTags = new GlobalTags(dataMap); - com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags); + com.linkedin.datahub.graphql.generated.GlobalTags graphQlGlobalTags = GlobalTagsMapper.map(globalTags, entityUrn); entity.setTags(graphQlGlobalTags); } private void mapDomains(@Nonnull MLPrimaryKey entity, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - entity.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + entity.setDomain(DomainAssociationMapper.map(domains, entity.getUrn())); } private void mapEditableProperties(MLPrimaryKey entity, DataMap dataMap) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java index 793fa7bcadb5cc..610bdb309114dd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/notebook/mappers/NotebookMapper.java @@ -7,12 +7,12 @@ import com.linkedin.common.Ownership; import com.linkedin.common.Status; import com.linkedin.common.SubTypes; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.ChartCell; import com.linkedin.datahub.graphql.generated.DataPlatform; -import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.NotebookCell; @@ -28,8 +28,9 @@ import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; -import com.linkedin.datahub.graphql.types.common.mappers.StringMapMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -54,23 +55,28 @@ public static Notebook map(EntityResponse response) { @Override public Notebook apply(EntityResponse response) { final Notebook convertedNotebook = new Notebook(); + Urn entityUrn = response.getUrn(); + convertedNotebook.setUrn(response.getUrn().toString()); convertedNotebook.setType(EntityType.NOTEBOOK); EnvelopedAspectMap aspectMap = response.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, convertedNotebook); mappingHelper.mapToResult(NOTEBOOK_KEY_ASPECT_NAME, this::mapNotebookKey); - mappingHelper.mapToResult(NOTEBOOK_INFO_ASPECT_NAME, this::mapNotebookInfo); + mappingHelper.mapToResult(NOTEBOOK_INFO_ASPECT_NAME, (entity, dataMap) -> this.mapNotebookInfo(entity, dataMap, entityUrn)); mappingHelper.mapToResult(NOTEBOOK_CONTENT_ASPECT_NAME, this::mapNotebookContent); mappingHelper.mapToResult(EDITABLE_NOTEBOOK_PROPERTIES_ASPECT_NAME, this::mapEditableNotebookProperties); - mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (notebook, dataMap) -> notebook.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (notebook, dataMap) -> notebook.setOwnership( + OwnershipMapper.map(new Ownership(dataMap), entityUrn) + )); mappingHelper.mapToResult(STATUS_ASPECT_NAME, (notebook, dataMap) -> notebook.setStatus(StatusMapper.map(new Status(dataMap)))); - mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (notebook, dataMap) -> notebook.setTags(GlobalTagsMapper.map(new GlobalTags(dataMap)))); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (notebook, dataMap) -> + notebook.setTags(GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn))); mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (notebook, dataMap) -> notebook.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); mappingHelper.mapToResult(DOMAINS_ASPECT_NAME, this::mapDomains); mappingHelper.mapToResult(SUB_TYPES_ASPECT_NAME, this::mapSubTypes); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (notebook, dataMap) -> - notebook.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap)))); + notebook.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, this::mapDataPlatformInstance); return mappingHelper.getResult(); } @@ -100,7 +106,7 @@ private void mapNotebookKey(@Nonnull Notebook notebook, @Nonnull DataMap dataMap notebook.setTool(notebookKey.getNotebookTool()); } - private void mapNotebookInfo(@Nonnull Notebook notebook, @Nonnull DataMap dataMap) { + private void mapNotebookInfo(@Nonnull Notebook notebook, @Nonnull DataMap dataMap, Urn entityUrn) { final com.linkedin.notebook.NotebookInfo gmsNotebookInfo = new com.linkedin.notebook.NotebookInfo(dataMap); final NotebookInfo notebookInfo = new NotebookInfo(); notebookInfo.setTitle(gmsNotebookInfo.getTitle()); @@ -112,7 +118,7 @@ private void mapNotebookInfo(@Nonnull Notebook notebook, @Nonnull DataMap dataMa } if (gmsNotebookInfo.hasCustomProperties()) { - notebookInfo.setCustomProperties(StringMapMapper.map(gmsNotebookInfo.getCustomProperties())); + notebookInfo.setCustomProperties(CustomPropertiesMapper.map(gmsNotebookInfo.getCustomProperties(), entityUrn)); } notebook.setInfo(notebookInfo); } @@ -188,10 +194,6 @@ private void mapEditableNotebookProperties(@Nonnull Notebook notebook, @Nonnull private void mapDomains(@Nonnull Notebook notebook, @Nonnull DataMap dataMap) { final Domains domains = new Domains(dataMap); // Currently we only take the first domain if it exists. - if (domains.getDomains().size() > 0) { - notebook.setDomain(Domain.builder() - .setType(EntityType.DOMAIN) - .setUrn(domains.getDomains().get(0).toString()).build()); - } + notebook.setDomain(DomainAssociationMapper.map(domains, notebook.getUrn())); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/GlobalTagsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/GlobalTagsMapper.java index 8acc40f23f98ec..f4d5f0a549a0ed 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/GlobalTagsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/GlobalTagsMapper.java @@ -2,31 +2,37 @@ import com.linkedin.common.GlobalTags; import com.linkedin.common.TagAssociation; +import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.Tag; -import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import javax.annotation.Nonnull; import java.util.stream.Collectors; -public class GlobalTagsMapper implements ModelMapper { +public class GlobalTagsMapper { public static final GlobalTagsMapper INSTANCE = new GlobalTagsMapper(); - public static com.linkedin.datahub.graphql.generated.GlobalTags map(@Nonnull final GlobalTags standardTags) { - return INSTANCE.apply(standardTags); + public static com.linkedin.datahub.graphql.generated.GlobalTags map( + @Nonnull final GlobalTags standardTags, + @Nonnull final Urn entityUrn + ) { + return INSTANCE.apply(standardTags, entityUrn); } - @Override - public com.linkedin.datahub.graphql.generated.GlobalTags apply(@Nonnull final GlobalTags input) { + public com.linkedin.datahub.graphql.generated.GlobalTags apply(@Nonnull final GlobalTags input, @Nonnull final Urn entityUrn) { final com.linkedin.datahub.graphql.generated.GlobalTags result = new com.linkedin.datahub.graphql.generated.GlobalTags(); - result.setTags(input.getTags().stream().map(this::mapTagAssociation).collect(Collectors.toList())); + result.setTags(input.getTags().stream().map(tag -> this.mapTagAssociation(tag, entityUrn)).collect(Collectors.toList())); return result; } - private com.linkedin.datahub.graphql.generated.TagAssociation mapTagAssociation(@Nonnull final TagAssociation input) { + private com.linkedin.datahub.graphql.generated.TagAssociation mapTagAssociation( + @Nonnull final TagAssociation input, + @Nonnull final Urn entityUrn + ) { final com.linkedin.datahub.graphql.generated.TagAssociation result = new com.linkedin.datahub.graphql.generated.TagAssociation(); final Tag resultTag = new Tag(); resultTag.setUrn(input.getTag().toString()); result.setTag(resultTag); + result.setAssociatedUrn(entityUrn.toString()); return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagMapper.java index 214a63b7df065c..43736b412b0045 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagMapper.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.types.tag.mappers; import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; import com.linkedin.data.template.GetMode; import com.linkedin.data.template.RecordTemplate; @@ -34,6 +35,7 @@ public static Tag map(@Nonnull final EntityResponse entityResponse) { @Override public Tag apply(@Nonnull final EntityResponse entityResponse) { final Tag result = new Tag(); + Urn entityUrn = entityResponse.getUrn(); result.setUrn(entityResponse.getUrn().toString()); result.setType(EntityType.TAG); @@ -45,7 +47,7 @@ public Tag apply(@Nonnull final EntityResponse entityResponse) { mappingHelper.mapToResult(TAG_KEY_ASPECT_NAME, this::mapTagKey); mappingHelper.mapToResult(TAG_PROPERTIES_ASPECT_NAME, this::mapTagProperties); mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (tag, dataMap) -> - tag.setOwnership(OwnershipMapper.map(new Ownership(dataMap)))); + tag.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); if (result.getProperties() != null && result.getProperties().getName() == null) { result.getProperties().setName(legacyName); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java index 264f4041f88cfb..b666bf5c60ed55 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/mappers/TagUpdateInputMapper.java @@ -42,7 +42,6 @@ public Collection apply( auditStamp.setActor(actor, SetMode.IGNORE_NULL); auditStamp.setTime(System.currentTimeMillis()); - // Creator is the owner. final Ownership ownership = new Ownership(); final Owner owner = new Owner(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaFieldBlameMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java similarity index 55% rename from datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaFieldBlameMapper.java rename to datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java index e7252b8a6369b4..9f51e0ae6fa7e8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaFieldBlameMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaBlameMapper.java @@ -6,13 +6,13 @@ import com.linkedin.datahub.graphql.generated.SchemaFieldBlame; import com.linkedin.datahub.graphql.generated.SchemaFieldChange; import com.linkedin.datahub.graphql.generated.SemanticVersionStruct; +import com.linkedin.datahub.graphql.types.timeline.utils.TimelineUtils; import com.linkedin.metadata.key.SchemaFieldKey; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeEvent; import com.linkedin.metadata.timeline.data.ChangeTransaction; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.util.Pair; -import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -24,14 +24,17 @@ import lombok.extern.slf4j.Slf4j; import org.apache.parquet.SemanticVersion; +import static com.linkedin.datahub.graphql.types.timeline.utils.TimelineUtils.*; + // Class for converting ChangeTransactions received from the Timeline API to SchemaFieldBlame structs for every schema // at every semantic version. @Slf4j -public class SchemaFieldBlameMapper { +public class SchemaBlameMapper { public static GetSchemaBlameResult map(List changeTransactions, @Nullable String versionCutoff) { if (changeTransactions.isEmpty()) { + log.debug("Change transactions are empty"); return null; } @@ -40,10 +43,6 @@ public static GetSchemaBlameResult map(List changeTransaction String latestSemanticVersionString = truncateSemanticVersion(changeTransactions.get(changeTransactions.size() - 1).getSemVer()); - long latestSemanticVersionTimestamp = changeTransactions.get(changeTransactions.size() - 1).getTimestamp(); - String latestVersionStamp = changeTransactions.get(changeTransactions.size() - 1).getVersionStamp(); - result.setLatestVersion( - new SemanticVersionStruct(latestSemanticVersionString, latestSemanticVersionTimestamp, latestVersionStamp)); String semanticVersionFilterString = versionCutoff == null ? latestSemanticVersionString : versionCutoff; Optional semanticVersionFilterOptional = createSemanticVersion(semanticVersionFilterString); @@ -54,7 +53,7 @@ public static GetSchemaBlameResult map(List changeTransaction SemanticVersion semanticVersionFilter = semanticVersionFilterOptional.get(); List reversedChangeTransactions = changeTransactions.stream() - .map(SchemaFieldBlameMapper::semanticVersionChangeTransactionPair) + .map(TimelineUtils::semanticVersionChangeTransactionPair) .filter(Optional::isPresent) .map(Optional::get) .filter(semanticVersionChangeTransactionPair -> @@ -69,13 +68,7 @@ public static GetSchemaBlameResult map(List changeTransaction result.setVersion( new SemanticVersionStruct(selectedSemanticVersion, selectedSemanticVersionTimestamp, selectedVersionStamp)); - List semanticVersionStructList = new ArrayList<>(); for (ChangeTransaction changeTransaction : reversedChangeTransactions) { - SemanticVersionStruct semanticVersionStruct = - new SemanticVersionStruct(truncateSemanticVersion(changeTransaction.getSemVer()), - changeTransaction.getTimestamp(), changeTransaction.getVersionStamp()); - semanticVersionStructList.add(semanticVersionStruct); - for (ChangeEvent changeEvent : changeTransaction.getChangeEvents()) { if (changeEvent.getCategory() != ChangeCategory.TECHNICAL_SCHEMA) { continue; @@ -115,71 +108,9 @@ public static GetSchemaBlameResult map(List changeTransaction .getChangeType() .equals(ChangeOperationType.REMOVE)) .collect(Collectors.toList())); - result.setSemanticVersionList(semanticVersionStructList); return result; } - private static Optional> semanticVersionChangeTransactionPair( - ChangeTransaction changeTransaction) { - Optional semanticVersion = createSemanticVersion(changeTransaction.getSemVer()); - return semanticVersion.map(version -> Pair.of(version, changeTransaction)); - } - - private static Optional createSemanticVersion(String semanticVersionString) { - String truncatedSemanticVersion = truncateSemanticVersion(semanticVersionString); - try { - SemanticVersion semanticVersion = SemanticVersion.parse(truncatedSemanticVersion); - return Optional.of(semanticVersion); - } catch (SemanticVersion.SemanticVersionParseException e) { - return Optional.empty(); - } - } - - // The SemanticVersion is currently returned from the ChangeTransactions in the format "x.y.z-computed". This function - // removes the suffix "computed". - private static String truncateSemanticVersion(String semanticVersion) { - String suffix = "-computed"; - return semanticVersion.endsWith(suffix) ? semanticVersion.substring(0, semanticVersion.lastIndexOf(suffix)) - : semanticVersion; - } - - private static SchemaFieldChange getLastSchemaFieldChange(ChangeEvent changeEvent, long timestamp, - String semanticVersion, String versionStamp) { - SchemaFieldChange schemaFieldChange = new SchemaFieldChange(); - schemaFieldChange.setTimestampMillis(timestamp); - schemaFieldChange.setLastSemanticVersion(truncateSemanticVersion(semanticVersion)); - schemaFieldChange.setChangeType( - ChangeOperationType.valueOf(ChangeOperationType.class, changeEvent.getOperation().toString())); - schemaFieldChange.setVersionStamp(versionStamp); - - String translatedChangeOperationType; - switch (changeEvent.getOperation()) { - case ADD: - translatedChangeOperationType = "Added"; - break; - case MODIFY: - translatedChangeOperationType = "Modified"; - break; - case REMOVE: - translatedChangeOperationType = "Removed"; - break; - default: - translatedChangeOperationType = "Unknown change made"; - log.warn(translatedChangeOperationType); - break; - } - - String suffix = "-computed"; - String translatedSemanticVersion = - semanticVersion.endsWith(suffix) ? semanticVersion.substring(0, semanticVersion.lastIndexOf(suffix)) - : semanticVersion; - - String lastSchemaFieldChange = String.format("%s in v%s", translatedChangeOperationType, translatedSemanticVersion); - schemaFieldChange.setLastSchemaFieldChange(lastSchemaFieldChange); - - return schemaFieldChange; - } - - private SchemaFieldBlameMapper() { + private SchemaBlameMapper() { } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaVersionListMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaVersionListMapper.java new file mode 100644 index 00000000000000..249957b1a12621 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/mappers/SchemaVersionListMapper.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.graphql.types.timeline.mappers; + +import com.linkedin.datahub.graphql.generated.GetSchemaVersionListResult; +import com.linkedin.datahub.graphql.generated.SemanticVersionStruct; +import com.linkedin.datahub.graphql.types.timeline.utils.TimelineUtils; +import com.linkedin.metadata.timeline.data.ChangeTransaction; +import com.linkedin.util.Pair; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.types.timeline.utils.TimelineUtils.*; + + +// Class for converting ChangeTransactions received from the Timeline API to list of schema versions. +@Slf4j +public class SchemaVersionListMapper { + + public static GetSchemaVersionListResult map(List changeTransactions) { + if (changeTransactions.isEmpty()) { + log.debug("Change transactions are empty"); + return null; + } + + GetSchemaVersionListResult result = new GetSchemaVersionListResult(); + + String latestSemanticVersionString = + truncateSemanticVersion(changeTransactions.get(changeTransactions.size() - 1).getSemVer()); + long latestSemanticVersionTimestamp = changeTransactions.get(changeTransactions.size() - 1).getTimestamp(); + String latestVersionStamp = changeTransactions.get(changeTransactions.size() - 1).getVersionStamp(); + result.setLatestVersion( + new SemanticVersionStruct(latestSemanticVersionString, latestSemanticVersionTimestamp, latestVersionStamp)); + + List reversedChangeTransactions = changeTransactions.stream() + .map(TimelineUtils::semanticVersionChangeTransactionPair) + .filter(Optional::isPresent) + .map(Optional::get) + .sorted(Collections.reverseOrder(Comparator.comparing(Pair::getFirst))) + .map(Pair::getSecond) + .collect(Collectors.toList()); + + List semanticVersionStructList = reversedChangeTransactions.stream() + .map(changeTransaction -> new SemanticVersionStruct(truncateSemanticVersion(changeTransaction.getSemVer()), + changeTransaction.getTimestamp(), changeTransaction.getVersionStamp())) + .collect(Collectors.toList()); + + result.setSemanticVersionList(semanticVersionStructList); + return result; + } + + private SchemaVersionListMapper() { + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/utils/TimelineUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/utils/TimelineUtils.java new file mode 100644 index 00000000000000..fcd77628045386 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/timeline/utils/TimelineUtils.java @@ -0,0 +1,79 @@ +package com.linkedin.datahub.graphql.types.timeline.utils; + +import com.linkedin.datahub.graphql.generated.ChangeOperationType; +import com.linkedin.datahub.graphql.generated.SchemaFieldChange; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeTransaction; +import com.linkedin.util.Pair; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.parquet.SemanticVersion; + + +@Slf4j +public class TimelineUtils { + + public static Optional> semanticVersionChangeTransactionPair( + ChangeTransaction changeTransaction) { + Optional semanticVersion = createSemanticVersion(changeTransaction.getSemVer()); + return semanticVersion.map(version -> Pair.of(version, changeTransaction)); + } + + public static Optional createSemanticVersion(String semanticVersionString) { + String truncatedSemanticVersion = truncateSemanticVersion(semanticVersionString); + try { + SemanticVersion semanticVersion = SemanticVersion.parse(truncatedSemanticVersion); + return Optional.of(semanticVersion); + } catch (SemanticVersion.SemanticVersionParseException e) { + return Optional.empty(); + } + } + + // The SemanticVersion is currently returned from the ChangeTransactions in the format "x.y.z-computed". This function + // removes the suffix "computed". + public static String truncateSemanticVersion(String semanticVersion) { + String suffix = "-computed"; + return semanticVersion.endsWith(suffix) ? semanticVersion.substring(0, semanticVersion.lastIndexOf(suffix)) + : semanticVersion; + } + + public static SchemaFieldChange getLastSchemaFieldChange(ChangeEvent changeEvent, long timestamp, + String semanticVersion, String versionStamp) { + SchemaFieldChange schemaFieldChange = new SchemaFieldChange(); + schemaFieldChange.setTimestampMillis(timestamp); + schemaFieldChange.setLastSemanticVersion(truncateSemanticVersion(semanticVersion)); + schemaFieldChange.setChangeType( + ChangeOperationType.valueOf(ChangeOperationType.class, changeEvent.getOperation().toString())); + schemaFieldChange.setVersionStamp(versionStamp); + + String translatedChangeOperationType; + switch (changeEvent.getOperation()) { + case ADD: + translatedChangeOperationType = "Added"; + break; + case MODIFY: + translatedChangeOperationType = "Modified"; + break; + case REMOVE: + translatedChangeOperationType = "Removed"; + break; + default: + translatedChangeOperationType = "Unknown change made"; + log.warn(translatedChangeOperationType); + break; + } + + String suffix = "-computed"; + String translatedSemanticVersion = + semanticVersion.endsWith(suffix) ? semanticVersion.substring(0, semanticVersion.lastIndexOf(suffix)) + : semanticVersion; + + String lastSchemaFieldChange = String.format("%s in v%s", translatedChangeOperationType, translatedSemanticVersion); + schemaFieldChange.setLastSchemaFieldChange(lastSchemaFieldChange); + + return schemaFieldChange; + } + + private TimelineUtils() { + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java deleted file mode 100644 index 745f3429f8e1ba..00000000000000 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.linkedin.datahub.graphql.types.usage; - -import com.datahub.authorization.ResourceSpec; -import com.linkedin.common.urn.Urn; -import com.linkedin.common.urn.UrnUtils; -import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.UsageStatsKey; -import com.linkedin.datahub.graphql.VersionedAspectKey; -import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; -import com.linkedin.datahub.graphql.types.LoadableType; -import com.linkedin.metadata.authorization.PoliciesConfig; -import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.usage.UsageClient; -import com.linkedin.usage.UsageQueryResult; -import graphql.execution.DataFetcherResult; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -public class UsageType implements LoadableType { - private final UsageClient _usageClient; - - public UsageType(final UsageClient usageClient) { - _usageClient = usageClient; - } - - @Override - public Class objectClass() { - return com.linkedin.datahub.graphql.generated.UsageQueryResult.class; - } - - @Override - public String name() { - return UsageType.class.getSimpleName(); - } - - /** - * Retrieves an list of aspects given a list of {@link VersionedAspectKey} structs. The list returned is expected to - * be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found. - * @param keys to retrieve - * @param context the {@link QueryContext} corresponding to the request. - */ - public List> batchLoad( - @Nonnull List keys, @Nonnull QueryContext context) throws Exception { - try { - return keys.stream().map(key -> { - Urn resourceUrn = UrnUtils.getUrn(key.getResource()); - if (!AuthorizationUtils.isAuthorized(context, - Optional.of(new ResourceSpec(resourceUrn.getEntityType(), key.getResource())), - PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE)) { - log.debug("User {} is not authorized to view usage information for dataset {}", context.getActorUrn(), - key.getResource()); - return DataFetcherResult.newResult().build(); - } - try { - UsageQueryResult usageQueryResult = - _usageClient.getUsageStats(key.getResource(), key.getRange(), context.getAuthentication()); - return DataFetcherResult.newResult().data( - UsageQueryResultMapper.map(usageQueryResult)).build(); - } catch (RemoteInvocationException | URISyntaxException e) { - throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", key.getResource()), e); - } - }).collect(Collectors.toList()); - } catch (Exception e) { - throw new RuntimeException("Failed to batch load Usage Stats", e); - } - } -} diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 6f61acb4362ca8..8f5e5e7a03f863 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -52,6 +52,11 @@ type PlatformPrivileges { """ generatePersonalAccessTokens: Boolean! + """ + Whether the user should be able to create new Domains + """ + createDomains: Boolean! + """ Whether the user should be able to manage Domains """ @@ -81,6 +86,21 @@ type PlatformPrivileges { Whether the user should be able to manage Glossaries """ manageGlossaries: Boolean! + + """ + Whether the user is able to manage user credentials + """ + manageUserCredentials: Boolean! + + """ + Whether the user should be able to create new Tags + """ + createTags: Boolean! + + """ + Whether the user should be able to create and delete all Tags + """ + manageTags: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index fee1a0df836ecf..2ed477155fc50f 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -162,7 +162,17 @@ type Query { """ Get whether or not not an entity exists """ - entityExisst(urn: String!): Boolean + entityExists(urn: String!): Boolean + + """ + Gets the current invite token. If the optional regenerate param is set to true, generate a new invite token. + """ + getNativeUserInviteToken: InviteToken + + """ + Gets an entity based on its urn + """ + entity(urn: String!): Entity } """ @@ -175,6 +185,11 @@ type Mutation { """ updateDataset(urn: String!, input: DatasetUpdateInput!): Dataset + """ + Update the metadata about a batch of Datasets + """ + updateDatasets(input: [BatchDatasetUpdateInput!]!): [Dataset] + """ Update the metadata about a particular Chart """ @@ -200,11 +215,26 @@ type Mutation { """ updateDataJob(urn: String!, input: DataJobUpdateInput!): DataJob + """ + Create a new tag. Requires the 'Manage Tags' or 'Create Tags' Platform Privilege. If a Tag with the provided ID already exists, + it will be overwritten. + """ + createTag( + "Inputs required to create a new Tag." + input: CreateTagInput!): String + """ Update the information about a particular Entity Tag """ updateTag(urn: String!, input: TagUpdateInput!): Tag + """ + Delete a Tag + """ + deleteTag( + "The urn of the Tag to delete" + urn: String!): Boolean + """ Set the hex color associated with an existing Tag """ @@ -235,16 +265,31 @@ type Mutation { """ addTags(input: AddTagsInput!): Boolean + """ + Add tags to multiple Entities or subresources + """ + batchAddTags(input: BatchAddTagsInput!): Boolean + """ Remove a tag from a particular Entity or subresource """ removeTag(input: TagAssociationInput!): Boolean + """ + Remove tags from multiple Entities or subresource + """ + batchRemoveTags(input: BatchRemoveTagsInput!): Boolean + """ Add a glossary term to a particular Entity or subresource """ addTerm(input: TermAssociationInput!): Boolean + """ + Add glossary terms to multiple Entities or subresource + """ + batchAddTerms(input: BatchAddTermsInput!): Boolean + """ Add multiple glossary terms to a particular Entity or subresource """ @@ -255,11 +300,21 @@ type Mutation { """ removeTerm(input: TermAssociationInput!): Boolean + """ + Remove glossary terms from multiple Entities or subresource + """ + batchRemoveTerms(input: BatchRemoveTermsInput!): Boolean + """ Add an owner to a particular Entity """ addOwner(input: AddOwnerInput!): Boolean + """ + Add owners to multiple Entities + """ + batchAddOwners(input: BatchAddOwnersInput!): Boolean + """ Add multiple owners to a particular Entity """ @@ -270,6 +325,11 @@ type Mutation { """ removeOwner(input: RemoveOwnerInput!): Boolean + """ + Remove owners from multiple Entities + """ + batchRemoveOwners(input: BatchRemoveOwnersInput!): Boolean + """ Add a link, or institutional memory, from a particular Entity """ @@ -316,16 +376,29 @@ type Mutation { createGroup(input: CreateGroupInput!): String """ - Create a new Domain. Returns the urn of the newly created Domain. Requires the Manage Domains privilege. If a domain with the provided ID already exists, + Create a new Domain. Returns the urn of the newly created Domain. Requires the 'Create Domains' or 'Manage Domains' Platform Privilege. If a Domain with the provided ID already exists, it will be overwritten. """ createDomain(input: CreateDomainInput!): String + """ + Delete a Domain + """ + deleteDomain( + "The urn of the Domain to delete" + urn: String!): Boolean + + """ Sets the Domain for a Dataset, Chart, Dashboard, Data Flow (Pipeline), or Data Job (Task). Returns true if the Domain was successfully added, or already exists. Requires the Edit Domains privilege for the Entity. """ setDomain(entityUrn: String!, domainUrn: String!): Boolean + """ + Set domain for multiple Entities + """ + batchSetDomain(input: BatchSetDomainInput!): Boolean + """ Sets the Domain for a Dataset, Chart, Dashboard, Data Flow (Pipeline), or Data Job (Task). Returns true if the Domain was successfully removed, or was already removed. Requires the Edit Domains privilege for an asset. """ @@ -338,6 +411,11 @@ type Mutation { "Input required to set deprecation for an Entity." input: UpdateDeprecationInput!): Boolean + """ + Updates the deprecation status for a batch of assets. + """ + batchUpdateDeprecation(input: BatchUpdateDeprecationInput!): Boolean + """ Update a particular Corp User's editable properties """ @@ -365,7 +443,7 @@ type Mutation { Input required to report an operation """ input: ReportOperationInput!): String - + """ Create a new GlossaryTerm. Returns the urn of the newly created GlossaryTerm. If a term with the provided ID already exists, it will be overwritten. """ @@ -390,6 +468,31 @@ type Mutation { Updates the name of the entity. """ updateName(input: UpdateNameInput!): Boolean + + """ + Add multiple related Terms to a Glossary Term to establish relationships + """ + addRelatedTerms(input: RelatedTermsInput!): Boolean + + """ + Remove multiple related Terms for a Glossary Term + """ + removeRelatedTerms(input: RelatedTermsInput!): Boolean + + """ + Generates an invite token that can be shared with prospective users to create their accounts. + """ + createNativeUserInviteToken: InviteToken + + """ + Generates a token that can be shared with existing native users to reset their credentials. + """ + createNativeUserResetToken(input: CreateNativeUserResetTokenInput!): ResetToken + + """ + Updates the soft deleted status for a batch of assets + """ + batchUpdateSoftDeleted(input: BatchUpdateSoftDeletedInput!): Boolean } """ @@ -572,6 +675,11 @@ input LineageInput { The number of results to be returned """ count: Int + + """ + Optional flag to not merge siblings in the response. They are merged by default. + """ + separateSiblings: Boolean } """ @@ -863,7 +971,7 @@ type Dataset implements EntityWithRelationships & Entity { """ The Domain associated with the Dataset """ - domain: Domain + domain: DomainAssociation """ Statistics about how this Dataset is used @@ -871,6 +979,11 @@ type Dataset implements EntityWithRelationships & Entity { """ usageStats(resource: String, range: TimeRange): UsageQueryResult + """ + Experimental - Summary operational & usage statistics about a Dataset + """ + statsSummary: DatasetStatsSummary + """ Profile Stats resource that retrieves the events in a previous unit of time in descending order If no start or end time are provided, the most recent events will be returned @@ -898,9 +1011,9 @@ type Dataset implements EntityWithRelationships & Entity { lineage(input: LineageInput!): EntityLineageResult """ - Experimental! The resolved health status of the Dataset + Experimental! The resolved health statuses of the Dataset """ - health: Health + health: [Health!] """ Schema metadata of the dataset @@ -965,6 +1078,25 @@ type Dataset implements EntityWithRelationships & Entity { History of datajob runs that either produced or consumed this dataset """ runs(start: Int, count: Int, direction: RelationshipDirection!): DataProcessInstanceResult + + """ + Metadata about the datasets siblings + """ + siblings: SiblingProperties +} + +""" +Metadata about the entity's siblings +""" +type SiblingProperties { + """ + If this entity is the primary sibling among the sibling set + """ + isPrimary: Boolean + """ + The sibling entities + """ + siblings: [Entity] } """ @@ -1065,12 +1197,12 @@ type VersionedDataset implements Entity { """ The Domain associated with the Dataset """ - domain: Domain + domain: DomainAssociation """ Experimental! The resolved health status of the Dataset """ - health: Health + health: [Health!] """ Schema metadata of the dataset @@ -1206,7 +1338,7 @@ type DatasetProperties { """ Custom properties of the Dataset """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ External URL associated with the Dataset @@ -1320,7 +1452,7 @@ type GlossaryTermInfo { """ Properties of the glossary term """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ Schema definition of glossary term @@ -1365,7 +1497,7 @@ type GlossaryTermProperties { """ Properties of the glossary term """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ Schema definition of glossary term @@ -1736,7 +1868,7 @@ type Container implements Entity { """ The Domain associated with the Dataset """ - domain: Domain + domain: DomainAssociation """ The deprecation status of the container @@ -1776,7 +1908,7 @@ type ContainerProperties { """ Custom properties of the Container """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ Native platform URL of the Container @@ -2060,6 +2192,11 @@ type Schema { The time at which the schema metadata information was created """ createdAt: Long + + """ + The time at which the schema metadata information was last ingested + """ + lastObserved: Long } """ @@ -2490,6 +2627,26 @@ type StringMapEntry { value: String } +""" +An entry in a custom properties map represented as a tuple +""" +type CustomPropertiesEntry { + """ + The key of the map entry + """ + key: String! + + """ + The value fo the map entry + """ + value: String + + """ + The urn of the entity this property came from for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! +} + """ The origin of Ownership metadata associated with a Metadata Entity """ @@ -2667,6 +2824,11 @@ type CorpUser implements Entity { """ relationships(input: RelationshipsInput!): EntityRelationshipsResult + """ + Whether or not this user is a native DataHub user + """ + isNativeUser: Boolean + """ Deprecated, use properties field instead Additional read only info about the corp user @@ -2948,6 +3110,7 @@ A DataHub Group entity, which represents a Person on the Metadata Entity Graph """ type CorpGroup implements Entity { """ + The primary key of the group """ urn: String! @@ -2982,6 +3145,11 @@ type CorpGroup implements Entity { """ relationships(input: RelationshipsInput!): EntityRelationshipsResult + """ + Origin info about this group. + """ + origin: Origin + """ Deprecated, use properties field instead Additional read only info about the group @@ -3111,6 +3279,11 @@ type Owner { Source information for the ownership """ source: OwnershipSource + + """ + Reference back to the owned urn for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! } """ @@ -3222,6 +3395,11 @@ type TagAssociation { The tag itself """ tag: Tag! + + """ + Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! } """ @@ -3261,6 +3439,11 @@ type GlossaryTermAssociation { The glossary term itself """ term: GlossaryTerm! + + """ + Reference back to the associated urn for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! } """ @@ -3428,6 +3611,23 @@ input DatasetUpdateInput { editableProperties: DatasetEditablePropertiesUpdate } +""" +Arguments provided to batch update Dataset entities +""" +input BatchDatasetUpdateInput { + + """ + Primary key of the Dataset to which the update will be applied + """ + urn: String! + + """ + Arguments provided to update the Dataset + """ + update: DatasetUpdateInput! +} + + """ Update to editable schema metadata of the dataset """ @@ -3566,6 +3766,26 @@ input TagUpdateInput { ownership: OwnershipUpdate } +""" +Input required to create a new Tag +""" +input CreateTagInput { + """ + Optional! A custom id to use as the primary key identifier for the Tag. If not provided, a random UUID will be generated as the id. + """ + id: String + + """ + Display name for the Tag + """ + name: String! + + """ + Optional description for the Tag + """ + description: String +} + """ An update for the ownership information for a Metadata Entity """ @@ -3710,7 +3930,7 @@ type Notebook implements Entity { """ The Domain associated with the Notebook """ - domain: Domain + domain: DomainAssociation """ The specific instance of the data platform that this entity belongs to @@ -3967,7 +4187,7 @@ type Dashboard implements EntityWithRelationships & Entity { """ The Domain associated with the Dashboard """ - domain: Domain + domain: DomainAssociation """ The specific instance of the data platform that this entity belongs to @@ -3984,6 +4204,16 @@ type Dashboard implements EntityWithRelationships & Entity { """ lineage(input: LineageInput!): EntityLineageResult + """ + Experimental (Subject to breaking change) -- Statistics about how this Dashboard is used + """ + usageStats(startTimeMillis: Long, endTimeMillis: Long, limit: Int): DashboardUsageQueryResult + + """ + Experimental - Summary operational & usage statistics about a Dashboard + """ + statsSummary: DashboardStatsSummary + """ Deprecated, use properties field instead Additional read only information about the dashboard @@ -4043,7 +4273,7 @@ type DashboardInfo { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ The time when this dashboard last refreshed @@ -4088,7 +4318,7 @@ type NotebookInfo { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ Captures information about who created/last modified/deleted this Notebook and when @@ -4124,7 +4354,7 @@ type DashboardProperties { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ The time when this dashboard last refreshed @@ -4230,13 +4460,20 @@ type Chart implements EntityWithRelationships & Entity { """ The Domain associated with the Chart """ - domain: Domain + domain: DomainAssociation """ The specific instance of the data platform that this entity belongs to """ dataPlatformInstance: DataPlatformInstance + """ + Not yet implemented. + + Experimental - Summary operational & usage statistics about a Chart + """ + statsSummary: ChartStatsSummary + """ Granular API for querying edges extending from this entity """ @@ -4310,7 +4547,7 @@ type ChartInfo { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ The time when this chart last refreshed @@ -4365,7 +4602,7 @@ type ChartProperties { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] """ The time when this chart last refreshed @@ -4567,7 +4804,7 @@ type DataFlow implements EntityWithRelationships & Entity { """ The Domain associated with the DataFlow """ - domain: Domain + domain: DomainAssociation """ The specific instance of the data platform that this entity belongs to @@ -4636,7 +4873,7 @@ type DataFlowInfo { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] } """ @@ -4666,7 +4903,7 @@ type DataFlowProperties { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] } """ @@ -4743,7 +4980,7 @@ type DataJob implements EntityWithRelationships & Entity { """ The Domain associated with the Data Job """ - domain: Domain + domain: DomainAssociation """ Granular API for querying edges extending from this entity @@ -4926,7 +5163,7 @@ type DataJobInfo { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] } """ @@ -4951,7 +5188,7 @@ type DataJobProperties { """ A list of platform specific metadata tuples """ - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] } """ @@ -5111,103 +5348,327 @@ type FieldUsageCounts { } """ -The duration of a fixed window of time +Experimental - subject to change. A summary of usage metrics about a Dataset. """ -enum WindowDuration { +type DatasetStatsSummary { """ - A one day window + The query count in the past 30 days """ - DAY + queryCountLast30Days: Int """ - A one week window + The unique user count in the past 30 days """ - WEEK + uniqueUserCountLast30Days: Int """ - A one month window + The top users in the past 30 days """ - MONTH + topUsersLast30Days: [CorpUser!] +} +""" +Information about individual user usage of a Dashboard +""" +type DashboardUserUsageCounts { """ - A one year window + The user of the Dashboard """ - YEAR + user: CorpUser + + """ + number of times dashboard has been viewed by the user + """ + viewsCount: Int + + """ + number of dashboard executions by the user + """ + executionsCount: Int + + """ + Normalized numeric metric representing user's dashboard usage + Higher value represents more usage + """ + usageCount: Int } """ -A time range used in fetching Dataset Usage statistics +The result of a dashboard usage query """ -enum TimeRange { +type DashboardUsageQueryResult { """ - Last day + A set of relevant time windows for use in displaying usage statistics """ - DAY + buckets: [DashboardUsageAggregation] """ - Last week + A set of rolled up aggregations about the dashboard usage """ - WEEK + aggregations: DashboardUsageQueryResultAggregations """ - Last month + A set of absolute dashboard usage metrics """ - MONTH + metrics: [DashboardUsageMetrics!] + +} +""" +A set of rolled up aggregations about the Dashboard usage +""" +type DashboardUsageQueryResultAggregations { """ - Last quarter + The count of unique Dashboard users within the queried time range """ - QUARTER + uniqueUserCount: Int """ - Last year + The specific per user usage counts within the queried time range """ - YEAR + users: [DashboardUserUsageCounts] """ - All time + The total number of dashboard views within the queried time range """ - ALL + viewsCount: Int + + """ + The total number of dashboard executions within the queried time range + """ + executionsCount: Int + } + """ -A Dataset Profile associated with a Dataset, containing profiling statistics about the Dataset +A set of absolute dashboard usage metrics """ -type DatasetProfile implements TimeSeriesAspect { - """ - The time at which the profile was reported - """ - timestampMillis: Long! +type DashboardUsageMetrics implements TimeSeriesAspect { + """ + The time at which the metrics were reported + """ + timestampMillis: Long! - """ - An optional row count of the Dataset - """ - rowCount: Long + """ + The total number of times dashboard has been favorited + FIXME: Qualifies as Popularity Metric rather than Usage Metric? + """ + favoritesCount: Int - """ - An optional column count of the Dataset - """ - columnCount: Long + """ + The total number of dashboard views + """ + viewsCount: Int - """ - An optional set of per field statistics obtained in the profile - """ - fieldProfiles: [DatasetFieldProfile!] + """ + The total number of dashboard execution + """ + executionsCount: Int + + """ + The time when this dashboard was last viewed + """ + lastViewed: Long - """ - Information about the partition that was profiled - """ - partitionSpec: PartitionSpec } """ -An individual Dataset Field Profile +An aggregation of Dashboard usage statistics """ -type DatasetFieldProfile { - """ - The standardized path of the field - """ - fieldPath: String! +type DashboardUsageAggregation { + """ + The time window start time + """ + bucket: Long + + """ + The time window span + """ + duration: WindowDuration + + """ + The resource urn associated with the usage information, eg a Dashboard urn + """ + resource: String + + """ + The rolled up usage metrics + """ + metrics: DashboardUsageAggregationMetrics +} + +""" +Rolled up metrics about Dashboard usage over time +""" +type DashboardUsageAggregationMetrics { + """ + The unique number of dashboard users within the time range + """ + uniqueUserCount: Int + + """ + The total number of dashboard views within the time range + """ + viewsCount: Int + + """ + The total number of dashboard executions within the time range + """ + executionsCount: Int + +} + +""" +Experimental - subject to change. A summary of usage metrics about a Dashboard. +""" +type DashboardStatsSummary { + """ + The total view count for the dashboard + """ + viewCount: Int + + """ + The view count in the last 30 days + """ + viewCountLast30Days: Int + + """ + The unique user count in the past 30 days + """ + uniqueUserCountLast30Days: Int + + """ + The top users in the past 30 days + """ + topUsersLast30Days: [CorpUser!] +} + + +""" +Experimental - subject to change. A summary of usage metrics about a Chart. +""" +type ChartStatsSummary { + """ + The total view count for the chart + """ + viewCount: Int + + """ + The view count in the last 30 days + """ + viewCountLast30Days: Int + + """ + The unique user count in the past 30 days + """ + uniqueUserCountLast30Days: Int + + """ + The top users in the past 30 days + """ + topUsersLast30Days: [CorpUser!] +} + + +""" +The duration of a fixed window of time +""" +enum WindowDuration { + """ + A one day window + """ + DAY + + """ + A one week window + """ + WEEK + + """ + A one month window + """ + MONTH + + """ + A one year window + """ + YEAR +} + +""" +A time range used in fetching Usage statistics +""" +enum TimeRange { + """ + Last day + """ + DAY + + """ + Last week + """ + WEEK + + """ + Last month + """ + MONTH + + """ + Last quarter + """ + QUARTER + + """ + Last year + """ + YEAR + + """ + All time + """ + ALL +} + +""" +A Dataset Profile associated with a Dataset, containing profiling statistics about the Dataset +""" +type DatasetProfile implements TimeSeriesAspect { + """ + The time at which the profile was reported + """ + timestampMillis: Long! + + """ + An optional row count of the Dataset + """ + rowCount: Long + + """ + An optional column count of the Dataset + """ + columnCount: Long + + """ + An optional set of per field statistics obtained in the profile + """ + fieldProfiles: [DatasetFieldProfile!] + + """ + Information about the partition that was profiled + """ + partitionSpec: PartitionSpec +} + +""" +An individual Dataset Field Profile +""" +type DatasetFieldProfile { + """ + The standardized path of the field + """ + fieldPath: String! """ The unique value count for the field across the Dataset @@ -5702,11 +6163,21 @@ enum AssertionStdOperator { """ START_WITH + """ + Value being asserted matches the regex value. + """ + REGEX_MATCH + """ Value being asserted is one of the array values """ IN + """ + Value being asserted is not in one of the array values. + """ + NOT_IN + """ Other """ @@ -6088,6 +6559,72 @@ enum SubResourceType { DATASET_FIELD } + +""" +Input provided when adding Terms to an asset +""" +input RelatedTermsInput { + """ + The Glossary Term urn to add or remove this relationship to/from + """ + urn: String! + + """ + The primary key of the Glossary Term to add or remove + """ + termUrns: [String!]! + + """ + The type of relationship we're adding or removing to/from for a Glossary Term + """ + relationshipType: TermRelationshipType! +} + +""" +A type of Metadata Entity sub resource +""" +enum TermRelationshipType { + """ + When a Term inherits from, or has an 'Is A' relationship with another Term + """ + isA + + """ + When a Term contains, or has a 'Has A' relationship with another Term + """ + hasA +} + +""" +Input provided when adding glossary terms to a batch of assets +""" +input BatchAddTermsInput { + """ + The primary key of the Glossary Terms + """ + termUrns: [String!]! + + """ + The target assets to attach the glossary terms to + """ + resources: [ResourceRefInput]! +} + +""" +Input provided when removing glossary terms from a batch of assets +""" +input BatchRemoveTermsInput { + """ + The primary key of the Glossary Terms + """ + termUrns: [String!]! + + """ + The target assets to remove the glossary terms from + """ + resources: [ResourceRefInput]! +} + """ Input provided when updating the association between a Metadata Entity and a Tag """ @@ -6138,6 +6675,56 @@ input AddTagsInput { subResource: String } +""" +Input provided when adding tags to a batch of assets +""" +input BatchAddTagsInput { + """ + The primary key of the Tags + """ + tagUrns: [String!]! + + """ + The target assets to attach the tags to + """ + resources: [ResourceRefInput!]! +} + +""" +Input provided when removing tags from a batch of assets +""" +input BatchRemoveTagsInput { + """ + The primary key of the Tags + """ + tagUrns: [String!]! + + """ + The target assets to remove the tags from + """ + resources: [ResourceRefInput]! +} + +""" +Reference to a resource to apply an action to +""" +input ResourceRefInput { + """ + The urn of the resource being referenced + """ + resourceUrn: String! + + """ + An optional type of a sub resource to attach the Tag to + """ + subResourceType: SubResourceType + + """ + An optional sub resource identifier to attach the Tag to + """ + subResource: String +} + """ Entities that are able to own other entities """ @@ -6213,6 +6800,36 @@ input AddOwnersInput { resourceUrn: String! } +""" +Input provided when adding owners to a batch of assets +""" +input BatchAddOwnersInput { + """ + The primary key of the owners + """ + owners: [OwnerInput!]! + + """ + The target assets to attach the owners to + """ + resources: [ResourceRefInput]! +} + +""" +Input provided when removing owners from a batch of assets +""" +input BatchRemoveOwnersInput { + """ + The primary key of the owners + """ + ownerUrns: [String!]! + + """ + The target assets to remove the owners from + """ + resources: [ResourceRefInput]! +} + """ Input provided when removing the association between a Metadata Entity and an user or group owner """ @@ -6343,6 +6960,47 @@ input UpdateDeprecationInput { note: String } + +""" +Input provided when updating the deprecation status for a batch of assets. +""" +input BatchUpdateDeprecationInput { + """ + Whether the Entity is marked as deprecated. + """ + deprecated: Boolean! + + """ + Optional - The time user plan to decommission this entity + """ + decommissionTime: Long + + """ + Optional - Additional information about the entity deprecation plan + """ + note: String + + """ + The target assets to attach the tags to + """ + resources: [ResourceRefInput]! +} + +""" +Input provided when adding tags to a batch of assets +""" +input BatchSetDomainInput { + """ + The primary key of the Domain, or null if the domain will be unset + """ + domainUrn: String + + """ + The target assets to attach the Domain + """ + resources: [ResourceRefInput!]! +} + """ Input provided when creating or updating an Access Policy """ @@ -6384,9 +7042,9 @@ input PolicyUpdateInput { } """ -Input required to add members to a DataHub group +Input required to add members to a native DataHub group """ -input AddGroupMembersInput { +input AddNativeGroupMembersInput { """ The group to add members to """ @@ -6399,18 +7057,48 @@ input AddGroupMembersInput { } """ -Input required to remove members from a DataHub group +Input required to add members to an external DataHub group +""" +input AddGroupMembersInput { + """ + The group to add members to + """ + groupUrn: String! + + """ + The members to add to the group + """ + userUrns: [String!]! +} + +""" +Input required to remove members from a native DataHub group +""" +input RemoveNativeGroupMembersInput { + """ + The group to remove members from + """ + groupUrn: String! + + """ + The members to remove from the group + """ + userUrns: [String!]! +} + +""" +Input required to remove members from an external DataHub group """ input RemoveGroupMembersInput { - """ - The group to remove members from - """ - groupUrn: String! + """ + The group to remove members from + """ + groupUrn: String! - """ - The members to remove from the group - """ - userUrns: [String!]! + """ + The members to remove from the group + """ + userUrns: [String!]! } """ @@ -7071,7 +7759,7 @@ type MLModel implements EntityWithRelationships & Entity { """ The Domain associated with the entity """ - domain: Domain + domain: DomainAssociation """ An additional set of of read write properties @@ -7162,7 +7850,7 @@ type MLModelGroup implements EntityWithRelationships & Entity { """ The Domain associated with the entity """ - domain: Domain + domain: DomainAssociation """ An additional set of of read write properties @@ -7271,7 +7959,7 @@ type MLFeature implements EntityWithRelationships & Entity { """ The Domain associated with the entity """ - domain: Domain + domain: DomainAssociation """ An additional set of of read write properties @@ -7321,7 +8009,7 @@ type MLModelProperties { groups: [MLModelGroup] - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] } type MLFeatureProperties { @@ -7428,7 +8116,7 @@ type MLPrimaryKey implements EntityWithRelationships & Entity { """ The Domain associated with the entity """ - domain: Domain + domain: DomainAssociation """ An additional set of of read write properties @@ -7535,7 +8223,7 @@ type MLFeatureTable implements EntityWithRelationships & Entity { """ The Domain associated with the entity """ - domain: Domain + domain: DomainAssociation """ An additional set of of read write properties @@ -7586,7 +8274,7 @@ description: String mlPrimaryKeys: [MLPrimaryKey] - customProperties: [StringMapEntry!] + customProperties: [CustomPropertiesEntry!] } type HyperParameterMap { @@ -7856,6 +8544,18 @@ type SubTypes { typeNames: [String!] } +type DomainAssociation { + """ + The domain related to the assocaited urn + """ + domain: Domain! + + """ + Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! +} + """ A domain, or a logical grouping of Metadata Entities """ @@ -8117,10 +8817,25 @@ enum HealthStatus { FAIL } +""" +The type of the health status +""" +enum HealthStatusType { + """ + Assertions status + """ + ASSERTIONS +} + """ The resolved Health of an Asset """ type Health { + """ + An enum representing the type of health indicator + """ + type: HealthStatusType! + """ An enum representing the resolved Health status of an Asset """ @@ -8151,3 +8866,81 @@ input StringMapEntryInput { """ value: String } + +""" +Token that allows users to sign up as a native user +""" +type InviteToken { + """ + The invite token + """ + inviteToken: String! +} + +""" +Input required to generate a password reset token for a native user. +""" +input CreateNativeUserResetTokenInput { + """ + The urn of the user to reset the password of + """ + userUrn: String! +} + +""" +Token that allows native users to reset their credentials +""" +type ResetToken { + """ + The reset token + """ + resetToken: String! +} + +""" +Carries information about where an entity originated from. +""" +type Origin { + """ + Where an entity originated from. Either NATIVE or EXTERNAL + """ + type: OriginType! + + """ + Only populated if type is EXTERNAL. The externalType of the entity, such as the name of the identity provider. + """ + externalType: String +} + +""" +Enum to define where an entity originated from. +""" +enum OriginType { + """ + The entity is native to DataHub. + """ + NATIVE + """ + The entity is external to DataHub. + """ + EXTERNAL + """ + The entity is of unknown origin. + """ + UNKNOWN +} + +""" +Input provided when updating the soft-deleted status for a batch of assets +""" +input BatchUpdateSoftDeletedInput { + """ + The urns of the assets to soft delete + """ + urns: [String!]! + + """ + Whether to mark the asset as soft-deleted (hidden) + """ + deleted: Boolean! +} diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql index 3bca51397416d7..3064a3b27165b6 100644 --- a/datahub-graphql-core/src/main/resources/ingestion.graphql +++ b/datahub-graphql-core/src/main/resources/ingestion.graphql @@ -66,6 +66,12 @@ extend type Mutation { Cancel a running execution request, provided the urn of the original execution request """ cancelIngestionExecutionRequest(input: CancelIngestionExecutionRequestInput!): String + + """ + Create a request to execute a test ingestion connection job + input: Input required for creating a test connection request + """ + createTestConnectionRequest(input: CreateTestConnectionRequestInput!): String } """ @@ -127,6 +133,29 @@ type ExecutionRequestResult { """ report: String + """ + A structured report for this Execution Request + """ + structuredReport: StructuredReport + +} + +""" +A flexible carrier for structured results of an execution request. +""" +type StructuredReport { + """ + The type of the structured report. (e.g. INGESTION_REPORT, TEST_CONNECTION_REPORT, etc.) + """ + type: String! + """ + The serialized value of the structured report + """ + serializedValue: String! + """ + The content-type of the serialized value (e.g. application/json, application/json;gzip etc.) + """ + contentType: String! } """ @@ -138,6 +167,11 @@ type ExecutionRequest { """ urn: String! + """ + Unique id for the execution request + """ + id: String! + """ Input provided when creating the Execution Request """ @@ -160,6 +194,16 @@ input CreateIngestionExecutionRequestInput { ingestionSourceUrn: String! } +""" +Input for creating a test connection request +""" +input CreateTestConnectionRequestInput { + """ + A JSON-encoded recipe + """ + recipe: String! +} + """ Input for cancelling an execution request input """ diff --git a/datahub-graphql-core/src/main/resources/timeline.graphql b/datahub-graphql-core/src/main/resources/timeline.graphql index 7c60cb7a29bede..e1757c315a16db 100644 --- a/datahub-graphql-core/src/main/resources/timeline.graphql +++ b/datahub-graphql-core/src/main/resources/timeline.graphql @@ -3,6 +3,11 @@ extend type Query { Returns the most recent changes made to each column in a dataset at each dataset version. """ getSchemaBlame(input: GetSchemaBlameInput!): GetSchemaBlameResult + + """ + Returns the list of schema versions for a dataset. + """ + getSchemaVersionList(input: GetSchemaVersionListInput!): GetSchemaVersionListResult } """ @@ -50,27 +55,19 @@ enum ChangeCategoryType { } """ -Input for getting schema changes computed at a specific version. +Input for getting list of schema versions. """ -input GetSchemaBlameInput { +input GetSchemaVersionListInput { """ The dataset urn """ datasetUrn: String! - """ - Categories to filter on. By default, will just be TECHNICAL_SCHEMA. - """ - categories: [ChangeCategoryType!] - """ - Changes after this version are not shown. If not provided, this is the latestVersion. - """ - version: String } """ Schema changes computed at a specific version. """ -type GetSchemaBlameResult { +type GetSchemaVersionListResult { """ Latest and current semantic version """ @@ -83,6 +80,31 @@ type GetSchemaBlameResult { All semantic versions. Absent when there are no versions. """ semanticVersionList: [SemanticVersionStruct!] +} + + +""" +Input for getting schema changes computed at a specific version. +""" +input GetSchemaBlameInput { + """ + The dataset urn + """ + datasetUrn: String! + """ + Changes after this version are not shown. If not provided, this is the latestVersion. + """ + version: String +} + +""" +Schema changes computed at a specific version. +""" +type GetSchemaBlameResult { + """ + Selected semantic version + """ + version: SemanticVersionStruct """ List of schema blame. Absent when there are no fields to return history for. """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java new file mode 100644 index 00000000000000..0a7d1c48a54dab --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java @@ -0,0 +1,190 @@ +package com.linkedin.datahub.graphql.resolvers.dashboard; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.dashboard.DashboardUsageStatistics; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Dashboard; +import com.linkedin.datahub.graphql.generated.DashboardStatsSummary; +import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; +import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.EnvelopedAspect; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.timeseries.GenericTable; +import com.linkedin.usage.UsageClient; +import com.linkedin.usage.UsageQueryResult; +import com.linkedin.usage.UsageQueryResultAggregations; +import com.linkedin.usage.UsageTimeRange; +import com.linkedin.usage.UserUsageCounts; +import com.linkedin.usage.UserUsageCountsArray; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*; + + +public class DashboardStatsSummaryTest { + + private static final Dashboard TEST_SOURCE = new Dashboard(); + private static final String TEST_DASHBOARD_URN = "urn:li:dashboard:(airflow,id)"; + private static final String TEST_USER_URN_1 = "urn:li:corpuser:test1"; + private static final String TEST_USER_URN_2 = "urn:li:corpuser:test2"; + + static { + TEST_SOURCE.setUrn(TEST_DASHBOARD_URN); + } + + @Test + public void testGetSuccess() throws Exception { + + TimeseriesAspectService mockClient = initTestAspectService(); + + // Execute resolver + DashboardStatsSummaryResolver resolver = new DashboardStatsSummaryResolver(mockClient); + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DashboardStatsSummary result = resolver.get(mockEnv).get(); + + // Validate Result + Assert.assertEquals((int) result.getViewCount(), 20); + Assert.assertEquals((int) result.getTopUsersLast30Days().size(), 2); + Assert.assertEquals((String) result.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2); + Assert.assertEquals((String) result.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1); + Assert.assertEquals((int) result.getUniqueUserCountLast30Days(), 2); + + // Validate the cache. -- First return a new result. + DashboardUsageStatistics newUsageStats = new DashboardUsageStatistics() + .setTimestampMillis(0L) + .setLastViewedAt(0L) + .setExecutionsCount(10) + .setFavoritesCount(5) + .setViewsCount(40); + EnvelopedAspect newResult = new EnvelopedAspect() + .setAspect(GenericRecordUtils.serializeAspect(newUsageStats)); + Filter filterForLatestStats = createUsageFilter(TEST_DASHBOARD_URN, null, null, false); + Mockito.when(mockClient.getAspectValues( + Mockito.eq(UrnUtils.getUrn(TEST_DASHBOARD_URN)), + Mockito.eq(Constants.DASHBOARD_ENTITY_NAME), + Mockito.eq(Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME), + Mockito.eq(null), + Mockito.eq(null), + Mockito.eq(1), + Mockito.eq(null), + Mockito.eq(filterForLatestStats) + )).thenReturn(ImmutableList.of(newResult)); + + // Then verify that the new result is _not_ returned (cache hit) + DashboardStatsSummary cachedResult = resolver.get(mockEnv).get(); + Assert.assertEquals((int) cachedResult.getViewCount(), 20); + Assert.assertEquals((int) cachedResult.getTopUsersLast30Days().size(), 2); + Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2); + Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1); + Assert.assertEquals((int) cachedResult.getUniqueUserCountLast30Days(), 2); + } + + @Test + public void testGetException() throws Exception { + // Init test UsageQueryResult + UsageQueryResult testResult = new UsageQueryResult(); + testResult.setAggregations(new UsageQueryResultAggregations() + .setUniqueUserCount(5) + .setTotalSqlQueries(10) + .setUsers(new UserUsageCountsArray( + ImmutableList.of( + new UserUsageCounts() + .setUser(UrnUtils.getUrn(TEST_USER_URN_1)) + .setUserEmail("test1@gmail.com") + .setCount(20), + new UserUsageCounts() + .setUser(UrnUtils.getUrn(TEST_USER_URN_2)) + .setUserEmail("test2@gmail.com") + .setCount(30) + ) + )) + ); + + UsageClient mockClient = Mockito.mock(UsageClient.class); + Mockito.when(mockClient.getUsageStats( + Mockito.eq(TEST_DASHBOARD_URN), + Mockito.eq(UsageTimeRange.MONTH), + Mockito.any(Authentication.class) + )).thenThrow(RuntimeException.class); + + // Execute resolver + DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient); + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // The resolver should NOT throw. + DatasetStatsSummary result = resolver.get(mockEnv).get(); + + // Summary should be null + Assert.assertNull(result); + } + + private TimeseriesAspectService initTestAspectService() { + + TimeseriesAspectService mockClient = Mockito.mock(TimeseriesAspectService.class); + + // Mock fetching the latest absolute (snapshot) statistics + DashboardUsageStatistics latestUsageStats = new DashboardUsageStatistics() + .setTimestampMillis(0L) + .setLastViewedAt(0L) + .setExecutionsCount(10) + .setFavoritesCount(5) + .setViewsCount(20); + EnvelopedAspect envelopedLatestStats = new EnvelopedAspect() + .setAspect(GenericRecordUtils.serializeAspect(latestUsageStats)); + + Filter filterForLatestStats = createUsageFilter(TEST_DASHBOARD_URN, null, null, false); + Mockito.when(mockClient.getAspectValues( + Mockito.eq(UrnUtils.getUrn(TEST_DASHBOARD_URN)), + Mockito.eq(Constants.DASHBOARD_ENTITY_NAME), + Mockito.eq(Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME), + Mockito.eq(null), + Mockito.eq(null), + Mockito.eq(1), + Mockito.eq(null), + Mockito.eq(filterForLatestStats) + )).thenReturn( + ImmutableList.of(envelopedLatestStats) + ); + + Mockito.when(mockClient.getAggregatedStats( + Mockito.eq(Constants.DASHBOARD_ENTITY_NAME), + Mockito.eq(Constants.DASHBOARD_USAGE_STATISTICS_ASPECT_NAME), + Mockito.any(), + Mockito.any(Filter.class), + Mockito.any() + )).thenReturn( + new GenericTable().setRows(new StringArrayArray( + new StringArray(ImmutableList.of( + TEST_USER_URN_1, "10", "20", "30", "1", "1", "1" + )), + new StringArray(ImmutableList.of( + TEST_USER_URN_2, "20", "30", "40", "1", "1", "1" + )) + )) + .setColumnNames(new StringArray()) + .setColumnTypes(new StringArray()) + ); + + return mockClient; + } + +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolverTest.java index 6dbe4e12bb6a42..ea9ab2a1b768b7 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetHealthResolverTest.java @@ -19,6 +19,7 @@ import com.linkedin.timeseries.GenericTable; import graphql.schema.DataFetchingEnvironment; import java.util.Collections; +import java.util.List; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -90,9 +91,10 @@ public void testGetSuccessHealthy() throws Exception { parentDataset.setUrn(TEST_DATASET_URN); Mockito.when(mockEnv.getSource()).thenReturn(parentDataset); - Health result = resolver.get(mockEnv).get(); + List result = resolver.get(mockEnv).get(); assertNotNull(result); - assertEquals(result.getStatus(), HealthStatus.PASS); + assertEquals(result.size(), 1); + assertEquals(result.get(0).getStatus(), HealthStatus.PASS); } @Test @@ -129,8 +131,8 @@ public void testGetSuccessNullHealth() throws Exception { parentDataset.setUrn(TEST_DATASET_URN); Mockito.when(mockEnv.getSource()).thenReturn(parentDataset); - Health result = resolver.get(mockEnv).get(); - assertNull(result); + List result = resolver.get(mockEnv).get(); + assertEquals(result.size(), 0); Mockito.verify(mockAspectService, Mockito.times(0)).getAggregatedStats( Mockito.any(), @@ -206,8 +208,8 @@ public void testGetSuccessUnhealthy() throws Exception { parentDataset.setUrn(TEST_DATASET_URN); Mockito.when(mockEnv.getSource()).thenReturn(parentDataset); - Health result = resolver.get(mockEnv).get(); - assertNotNull(result); - assertEquals(result.getStatus(), HealthStatus.FAIL); + List result = resolver.get(mockEnv).get(); + assertEquals(result.size(), 1); + assertEquals(result.get(0).getStatus(), HealthStatus.FAIL); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java new file mode 100644 index 00000000000000..bd3edf65bf7ad5 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java @@ -0,0 +1,137 @@ +package com.linkedin.datahub.graphql.resolvers.dataset; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Dataset; +import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; +import com.linkedin.usage.UsageClient; +import com.linkedin.usage.UsageQueryResult; +import com.linkedin.usage.UsageQueryResultAggregations; +import com.linkedin.usage.UsageTimeRange; +import com.linkedin.usage.UserUsageCounts; +import com.linkedin.usage.UserUsageCountsArray; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class DatasetStatsSummaryResolverTest { + + private static final Dataset TEST_SOURCE = new Dataset(); + private static final String TEST_DATASET_URN = "urn:li:dataset:(urn:li:dataPlatform:hive,test,PROD)"; + private static final String TEST_USER_URN_1 = "urn:li:corpuser:test1"; + private static final String TEST_USER_URN_2 = "urn:li:corpuser:test2"; + + static { + TEST_SOURCE.setUrn(TEST_DATASET_URN); + } + + @Test + public void testGetSuccess() throws Exception { + // Init test UsageQueryResult + UsageQueryResult testResult = new UsageQueryResult(); + testResult.setAggregations(new UsageQueryResultAggregations() + .setUniqueUserCount(5) + .setTotalSqlQueries(10) + .setUsers(new UserUsageCountsArray( + ImmutableList.of( + new UserUsageCounts() + .setUser(UrnUtils.getUrn(TEST_USER_URN_1)) + .setUserEmail("test1@gmail.com") + .setCount(20), + new UserUsageCounts() + .setUser(UrnUtils.getUrn(TEST_USER_URN_2)) + .setUserEmail("test2@gmail.com") + .setCount(30) + ) + )) + ); + + UsageClient mockClient = Mockito.mock(UsageClient.class); + Mockito.when(mockClient.getUsageStats( + Mockito.eq(TEST_DATASET_URN), + Mockito.eq(UsageTimeRange.MONTH), + Mockito.any(Authentication.class) + )).thenReturn(testResult); + + // Execute resolver + DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient); + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DatasetStatsSummary result = resolver.get(mockEnv).get(); + + // Validate Result + Assert.assertEquals((int) result.getQueryCountLast30Days(), 10); + Assert.assertEquals((int) result.getTopUsersLast30Days().size(), 2); + Assert.assertEquals((String) result.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2); + Assert.assertEquals((String) result.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1); + Assert.assertEquals((int) result.getUniqueUserCountLast30Days(), 5); + + // Validate the cache. -- First return a new result. + UsageQueryResult newResult = new UsageQueryResult(); + newResult.setAggregations(new UsageQueryResultAggregations()); + Mockito.when(mockClient.getUsageStats( + Mockito.eq(TEST_DATASET_URN), + Mockito.eq(UsageTimeRange.MONTH), + Mockito.any(Authentication.class) + )).thenReturn(newResult); + + // Then verify that the new result is _not_ returned (cache hit) + DatasetStatsSummary cachedResult = resolver.get(mockEnv).get(); + Assert.assertEquals((int) cachedResult.getQueryCountLast30Days(), 10); + Assert.assertEquals((int) cachedResult.getTopUsersLast30Days().size(), 2); + Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(0).getUrn(), TEST_USER_URN_2); + Assert.assertEquals((String) cachedResult.getTopUsersLast30Days().get(1).getUrn(), TEST_USER_URN_1); + Assert.assertEquals((int) cachedResult.getUniqueUserCountLast30Days(), 5); + } + + @Test + public void testGetException() throws Exception { + // Init test UsageQueryResult + UsageQueryResult testResult = new UsageQueryResult(); + testResult.setAggregations(new UsageQueryResultAggregations() + .setUniqueUserCount(5) + .setTotalSqlQueries(10) + .setUsers(new UserUsageCountsArray( + ImmutableList.of( + new UserUsageCounts() + .setUser(UrnUtils.getUrn(TEST_USER_URN_1)) + .setUserEmail("test1@gmail.com") + .setCount(20), + new UserUsageCounts() + .setUser(UrnUtils.getUrn(TEST_USER_URN_2)) + .setUserEmail("test2@gmail.com") + .setCount(30) + ) + )) + ); + + UsageClient mockClient = Mockito.mock(UsageClient.class); + Mockito.when(mockClient.getUsageStats( + Mockito.eq(TEST_DATASET_URN), + Mockito.eq(UsageTimeRange.MONTH), + Mockito.any(Authentication.class) + )).thenThrow(RuntimeException.class); + + // Execute resolver + DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient); + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // The resolver should NOT throw. + DatasetStatsSummary result = resolver.get(mockEnv).get(); + + // Summary should be null + Assert.assertNull(result); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java new file mode 100644 index 00000000000000..2fe927100d55cb --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/delete/BatchUpdateSoftDeletedResolverTest.java @@ -0,0 +1,219 @@ +package com.linkedin.datahub.graphql.resolvers.delete; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchUpdateSoftDeletedInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateSoftDeletedResolver; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchUpdateSoftDeletedResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + + @Test + public void testGetSuccessNoExistingStatus() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.STATUS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.STATUS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + BatchUpdateSoftDeletedResolver resolver = new BatchUpdateSoftDeletedResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateSoftDeletedInput input = new BatchUpdateSoftDeletedInput(ImmutableList.of(TEST_ENTITY_URN_1, TEST_ENTITY_URN_2), true); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Status newStatus = new Status().setRemoved(true); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.STATUS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newStatus)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.STATUS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newStatus)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetSuccessExistingStatus() throws Exception { + final Status originalStatus = new Status().setRemoved(true); + + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.STATUS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalStatus); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.STATUS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalStatus); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + BatchUpdateSoftDeletedResolver resolver = new BatchUpdateSoftDeletedResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateSoftDeletedInput input = new BatchUpdateSoftDeletedInput(ImmutableList.of(TEST_ENTITY_URN_1, TEST_ENTITY_URN_2), false); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Status newStatus = new Status().setRemoved(false); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.STATUS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newStatus)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.STATUS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newStatus)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.STATUS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.STATUS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + BatchUpdateSoftDeletedResolver resolver = new BatchUpdateSoftDeletedResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateSoftDeletedInput input = new BatchUpdateSoftDeletedInput(ImmutableList.of(TEST_ENTITY_URN_1, TEST_ENTITY_URN_2), false); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchUpdateSoftDeletedResolver resolver = new BatchUpdateSoftDeletedResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateSoftDeletedInput input = new BatchUpdateSoftDeletedInput(ImmutableList.of(TEST_ENTITY_URN_1, TEST_ENTITY_URN_2), false); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchUpdateSoftDeletedResolver resolver = new BatchUpdateSoftDeletedResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchUpdateSoftDeletedInput input = new BatchUpdateSoftDeletedInput(ImmutableList.of(TEST_ENTITY_URN_1, TEST_ENTITY_URN_2), false); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java new file mode 100644 index 00000000000000..49c24770333c73 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/deprecation/BatchUpdateDeprecationResolverTest.java @@ -0,0 +1,238 @@ +package com.linkedin.datahub.graphql.resolvers.deprecation; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Deprecation; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchUpdateDeprecationInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateDeprecationResolver; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchUpdateDeprecationResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + + @Test + public void testGetSuccessNoExistingDeprecation() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DEPRECATION_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DEPRECATION_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + BatchUpdateDeprecationResolver resolver = new BatchUpdateDeprecationResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateDeprecationInput input = new BatchUpdateDeprecationInput(true, 0L, "test", ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Deprecation newDeprecation = new Deprecation() + .setDeprecated(true) + .setNote("test") + .setDecommissionTime(0L) + .setActor(UrnUtils.getUrn("urn:li:corpuser:test")); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.DEPRECATION_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.DEPRECATION_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetSuccessExistingDeprecation() throws Exception { + final Deprecation originalDeprecation = new Deprecation() + .setDeprecated(false) + .setNote("") + .setActor(UrnUtils.getUrn("urn:li:corpuser:test")); + + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DEPRECATION_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalDeprecation); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DEPRECATION_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalDeprecation); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + BatchUpdateDeprecationResolver resolver = new BatchUpdateDeprecationResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateDeprecationInput input = new BatchUpdateDeprecationInput(true, 1L, "test", ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Deprecation newDeprecation = new Deprecation() + .setDeprecated(true) + .setNote("test") + .setDecommissionTime(1L) + .setActor(UrnUtils.getUrn("urn:li:corpuser:test")); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.DEPRECATION_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.DEPRECATION_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newDeprecation)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DEPRECATION_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DEPRECATION_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + BatchUpdateDeprecationResolver resolver = new BatchUpdateDeprecationResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateDeprecationInput input = new BatchUpdateDeprecationInput(true, 1L, "test", ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchUpdateDeprecationResolver resolver = new BatchUpdateDeprecationResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchUpdateDeprecationInput input = new BatchUpdateDeprecationInput(true, 1L, "test", ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchUpdateDeprecationResolver resolver = new BatchUpdateDeprecationResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchUpdateDeprecationInput input = new BatchUpdateDeprecationInput(true, 1L, "test", ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java new file mode 100644 index 00000000000000..756e085593c26c --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/BatchSetDomainResolverTest.java @@ -0,0 +1,344 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchSetDomainInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchSetDomainResolver; +import com.linkedin.domain.Domains; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchSetDomainResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_DOMAIN_1_URN = "urn:li:domain:test-id-1"; + private static final String TEST_DOMAIN_2_URN = "urn:li:domain:test-id-2"; + + @Test + public void testGetSuccessNoExistingDomains() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_2_URN))).thenReturn(true); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchSetDomainInput input = new BatchSetDomainInput(TEST_DOMAIN_2_URN, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Domains newDomains = new Domains().setDomains(new UrnArray(ImmutableList.of( + Urn.createFromString(TEST_DOMAIN_2_URN) + ))); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.DOMAINS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newDomains)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.DOMAINS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newDomains)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_DOMAIN_2_URN)) + ); + } + + @Test + public void testGetSuccessExistingDomains() throws Exception { + final Domains originalDomain = new Domains().setDomains(new UrnArray(ImmutableList.of( + Urn.createFromString(TEST_DOMAIN_1_URN)))); + + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalDomain); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalDomain); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_2_URN))).thenReturn(true); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchSetDomainInput input = new BatchSetDomainInput(TEST_DOMAIN_2_URN, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Domains newDomains = new Domains().setDomains(new UrnArray(ImmutableList.of( + Urn.createFromString(TEST_DOMAIN_2_URN) + ))); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.DOMAINS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newDomains)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.DOMAINS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newDomains)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_DOMAIN_2_URN)) + ); + } + + @Test + public void testGetSuccessUnsetDomains() throws Exception { + final Domains originalDomain = new Domains().setDomains(new UrnArray(ImmutableList.of( + Urn.createFromString(TEST_DOMAIN_1_URN)))); + + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalDomain); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalDomain); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_2_URN))).thenReturn(true); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchSetDomainInput input = new BatchSetDomainInput(null, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final Domains newDomains = new Domains().setDomains(new UrnArray(ImmutableList.of())); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.DOMAINS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newDomains)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.DOMAINS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newDomains)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailureDomainDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_1_URN))).thenReturn(false); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchSetDomainInput input = new BatchSetDomainInput(null, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.DOMAINS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_DOMAIN_1_URN))).thenReturn(true); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchSetDomainInput input = new BatchSetDomainInput(null, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchSetDomainInput input = new BatchSetDomainInput(null, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchSetDomainResolver resolver = new BatchSetDomainResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchSetDomainInput input = new BatchSetDomainInput(null, ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java new file mode 100644 index 00000000000000..1c450b0e85424d --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class DeleteDomainResolverTest { + + private static final String TEST_URN = "urn:li:domain:test-id"; + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + DeleteDomainResolver resolver = new DeleteDomainResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockClient, Mockito.times(1)).deleteEntity( + Mockito.eq(Urn.createFromString(TEST_URN)), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + DeleteDomainResolver resolver = new DeleteDomainResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).deleteEntity( + Mockito.any(), + Mockito.any(Authentication.class)); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java new file mode 100644 index 00000000000000..451faf9bc8e382 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/AddRelatedTermsResolverTest.java @@ -0,0 +1,255 @@ +package com.linkedin.datahub.graphql.resolvers.glossary; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.RelatedTermsInput; +import com.linkedin.datahub.graphql.generated.TermRelationshipType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.ExecutionException; + +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.testng.Assert.*; + + +public class AddRelatedTermsResolverTest { + + private static final String TEST_ENTITY_URN = "urn:li:glossaryTerm:test-id-0"; + private static final String TEST_TERM_1_URN = "urn:li:glossaryTerm:test-id-1"; + private static final String TEST_TERM_2_URN = "urn:li:glossaryTerm:test-id-2"; + private static final String DATASET_URN = "urn:li:dataset:(test,test,test)"; + + private EntityService setUpService() { + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), + Mockito.eq(Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + return mockService; + } + + @Test + public void testGetSuccessIsRelatedNonExistent() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_2_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), TermRelationshipType.isA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TERM_1_URN)) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TERM_2_URN)) + ); + } + + @Test + public void testGetSuccessHasRelatedNonExistent() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_2_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TERM_1_URN)) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TERM_2_URN)) + ); + } + + @Test + public void testGetFailAddSelfAsRelatedTerm() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_ENTITY_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailAddNonTermAsRelatedTerm() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + DATASET_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailAddNonExistentTermAsRelatedTerm() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(false); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailAddToNonExistentUrn() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailAddToNonTerm() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(DATASET_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(DATASET_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testFailNoPermissions() throws Exception { + EntityService mockService = setUpService(); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_2_URN))).thenReturn(true); + + AddRelatedTermsResolver resolver = new AddRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), TermRelationshipType.isA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java new file mode 100644 index 00000000000000..6a704c2b61c127 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/RemoveRelatedTermsResolverTest.java @@ -0,0 +1,166 @@ +package com.linkedin.datahub.graphql.resolvers.glossary; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlossaryTermUrnArray; +import com.linkedin.common.urn.GlossaryTermUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.RelatedTermsInput; +import com.linkedin.datahub.graphql.generated.TermRelationshipType; +import com.linkedin.glossary.GlossaryRelatedTerms; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.concurrent.ExecutionException; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +public class RemoveRelatedTermsResolverTest { + + private static final String TEST_ENTITY_URN = "urn:li:glossaryTerm:test-id-0"; + private static final String TEST_TERM_1_URN = "urn:li:glossaryTerm:test-id-1"; + private static final String TEST_TERM_2_URN = "urn:li:glossaryTerm:test-id-2"; + + @Test + public void testGetSuccessIsA() throws Exception { + GlossaryTermUrn term1Urn = GlossaryTermUrn.createFromString(TEST_TERM_1_URN); + GlossaryTermUrn term2Urn = GlossaryTermUrn.createFromString(TEST_TERM_2_URN); + final GlossaryRelatedTerms relatedTerms = new GlossaryRelatedTerms(); + relatedTerms.setIsRelatedTerms(new GlossaryTermUrnArray(Arrays.asList(term1Urn, term2Urn))); + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), + Mockito.eq(Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(relatedTerms); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + RemoveRelatedTermsResolver resolver = new RemoveRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.isA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + } + + @Test + public void testGetSuccessHasA() throws Exception { + GlossaryTermUrn term1Urn = GlossaryTermUrn.createFromString(TEST_TERM_1_URN); + GlossaryTermUrn term2Urn = GlossaryTermUrn.createFromString(TEST_TERM_2_URN); + final GlossaryRelatedTerms relatedTerms = new GlossaryRelatedTerms(); + relatedTerms.setHasRelatedTerms(new GlossaryTermUrnArray(Arrays.asList(term1Urn, term2Urn))); + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), + Mockito.eq(Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(relatedTerms); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + RemoveRelatedTermsResolver resolver = new RemoveRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + } + + @Test + public void testFailAspectDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), + Mockito.eq(Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + RemoveRelatedTermsResolver resolver = new RemoveRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.hasA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testFailNoPermissions() throws Exception { + GlossaryTermUrn term1Urn = GlossaryTermUrn.createFromString(TEST_TERM_1_URN); + GlossaryTermUrn term2Urn = GlossaryTermUrn.createFromString(TEST_TERM_2_URN); + final GlossaryRelatedTerms relatedTerms = new GlossaryRelatedTerms(); + relatedTerms.setIsRelatedTerms(new GlossaryTermUrnArray(Arrays.asList(term1Urn, term2Urn))); + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN)), + Mockito.eq(Constants.GLOSSARY_RELATED_TERM_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(relatedTerms); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN))).thenReturn(true); + + RemoveRelatedTermsResolver resolver = new RemoveRelatedTermsResolver(mockService); + + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + RelatedTermsInput input = new RelatedTermsInput(TEST_ENTITY_URN, ImmutableList.of( + TEST_TERM_1_URN + ), TermRelationshipType.isA); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + Mockito.verify(mockService, Mockito.times(0)).exists( + Mockito.eq(Urn.createFromString(TEST_ENTITY_URN)) + ); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java index b704264b1c308b..e3edfe0efe1342 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java @@ -7,6 +7,7 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateNameInput; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; +import com.linkedin.domain.DomainProperties; import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.glossary.GlossaryTermInfo; @@ -29,8 +30,10 @@ public class UpdateNameResolverTest { private static final String NEW_NAME = "New Name"; private static final String TERM_URN = "urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"; private static final String NODE_URN = "urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"; + private static final String DOMAIN_URN = "urn:li:domain:22225397daf94708a8822b8106cfd451"; private static final UpdateNameInput INPUT = new UpdateNameInput(NEW_NAME, TERM_URN); private static final UpdateNameInput INPUT_FOR_NODE = new UpdateNameInput(NEW_NAME, NODE_URN); + private static final UpdateNameInput INPUT_FOR_DOMAIN = new UpdateNameInput(NEW_NAME, DOMAIN_URN); private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); private MetadataChangeProposal setupTests(DataFetchingEnvironment mockEnv, EntityService mockService) throws Exception { @@ -111,6 +114,43 @@ public void testGetSuccessForNode() throws Exception { ); } + @Test + public void testGetSuccessForDomain() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + Mockito.when(mockService.exists(Urn.createFromString(DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT_FOR_DOMAIN); + + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + final String name = "test name"; + Mockito.when(mockService.getAspect( + Urn.createFromString(DOMAIN_URN), + Constants.DOMAIN_PROPERTIES_ASPECT_NAME, + 0)) + .thenReturn(new DomainProperties().setName(name)); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString(DOMAIN_URN)); + proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME); + DomainProperties properties = new DomainProperties(); + properties.setName(NEW_NAME); + proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(properties)); + proposal.setChangeType(ChangeType.UPSERT); + + UpdateNameResolver resolver = new UpdateNameResolver(mockService); + + assertTrue(resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.any() + ); + } + @Test public void testGetFailureEntityDoesNotExist() throws Exception { EntityService mockService = Mockito.mock(EntityService.class); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolverTest.java new file mode 100644 index 00000000000000..a20c84d11ba9fd --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/AddGroupMembersResolverTest.java @@ -0,0 +1,85 @@ +package com.linkedin.datahub.graphql.resolvers.group; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.Origin; +import com.linkedin.common.OriginType; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AddGroupMembersInput; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class AddGroupMembersResolverTest { + private static final String GROUP_URN_STRING = "urn:li:corpGroup:testNewGroup"; + private static final String USER_URN_STRING = "urn:li:corpuser:test"; + + private static Urn _groupUrn; + + private GroupService _groupService; + private AddGroupMembersResolver _resolver; + private DataFetchingEnvironment _dataFetchingEnvironment; + private Authentication _authentication; + + @BeforeMethod + public void setupTest() throws Exception { + _groupUrn = Urn.createFromString(GROUP_URN_STRING); + + _groupService = mock(GroupService.class); + _dataFetchingEnvironment = mock(DataFetchingEnvironment.class); + _authentication = mock(Authentication.class); + AddGroupMembersInput input = new AddGroupMembersInput(); + input.setGroupUrn(GROUP_URN_STRING); + input.setUserUrns(new ArrayList<>(Collections.singleton(USER_URN_STRING))); + + _resolver = new AddGroupMembersResolver(_groupService); + + when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input); + } + + @Test + public void testFailsCannotManageUsersAndGroups() { + QueryContext mockContext = getMockDenyContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testFailsExternalGroup() { + Origin groupOrigin = new Origin(); + groupOrigin.setType(OriginType.EXTERNAL); + + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(mockContext.getActorUrn()).thenReturn(USER_URN_STRING); + when(_groupService.groupExists(any())).thenReturn(true); + when(_groupService.getGroupOrigin(eq(_groupUrn))).thenReturn(groupOrigin); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testPassesNativeGroup() throws Exception { + Origin groupOrigin = new Origin(); + groupOrigin.setType(OriginType.NATIVE); + + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(mockContext.getActorUrn()).thenReturn(USER_URN_STRING); + when(_groupService.groupExists(any())).thenReturn(true); + when(_groupService.getGroupOrigin(eq(_groupUrn))).thenReturn(groupOrigin); + + _resolver.get(_dataFetchingEnvironment).join(); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolverTest.java new file mode 100644 index 00000000000000..876de633bd6562 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/CreateGroupResolverTest.java @@ -0,0 +1,54 @@ +package com.linkedin.datahub.graphql.resolvers.group; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateGroupInput; +import graphql.schema.DataFetchingEnvironment; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class CreateGroupResolverTest { + private static final String GROUP_ID = "id"; + + private GroupService _groupService; + private CreateGroupResolver _resolver; + private DataFetchingEnvironment _dataFetchingEnvironment; + private Authentication _authentication; + private CreateGroupInput _input; + + @BeforeMethod + public void setupTest() { + _groupService = mock(GroupService.class); + _dataFetchingEnvironment = mock(DataFetchingEnvironment.class); + _authentication = mock(Authentication.class); + _input = new CreateGroupInput(); + _input.setId(GROUP_ID); + + _resolver = new CreateGroupResolver(_groupService); + } + + @Test + public void testFailsCannotManageUsersAndGroups() { + QueryContext mockContext = getMockDenyContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testPasses() throws Exception { + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(_input); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(_groupService.groupExists(any())).thenReturn(false); + + _resolver.get(_dataFetchingEnvironment).join(); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolverTest.java new file mode 100644 index 00000000000000..73b0be96fce176 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/group/RemoveGroupMembersResolverTest.java @@ -0,0 +1,85 @@ +package com.linkedin.datahub.graphql.resolvers.group; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.Origin; +import com.linkedin.common.OriginType; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.RemoveGroupMembersInput; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class RemoveGroupMembersResolverTest { + private static final String GROUP_URN_STRING = "urn:li:corpGroup:testNewGroup"; + private static final String USER_URN_STRING = "urn:li:corpuser:test"; + + private static Urn _groupUrn; + + private GroupService _groupService; + private RemoveGroupMembersResolver _resolver; + private DataFetchingEnvironment _dataFetchingEnvironment; + private Authentication _authentication; + + @BeforeMethod + public void setupTest() throws Exception { + _groupUrn = Urn.createFromString(GROUP_URN_STRING); + + _groupService = mock(GroupService.class); + _dataFetchingEnvironment = mock(DataFetchingEnvironment.class); + _authentication = mock(Authentication.class); + RemoveGroupMembersInput input = new RemoveGroupMembersInput(); + input.setGroupUrn(GROUP_URN_STRING); + input.setUserUrns(new ArrayList<>(Collections.singleton(USER_URN_STRING))); + + _resolver = new RemoveGroupMembersResolver(_groupService); + + when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input); + } + + @Test + public void testFailsCannotManageUsersAndGroups() { + QueryContext mockContext = getMockDenyContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testFailsExternalGroup() { + Origin groupOrigin = new Origin(); + groupOrigin.setType(OriginType.EXTERNAL); + + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(mockContext.getActorUrn()).thenReturn(USER_URN_STRING); + when(_groupService.groupExists(any())).thenReturn(true); + when(_groupService.getGroupOrigin(eq(_groupUrn))).thenReturn(groupOrigin); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testPassesNativeGroup() throws Exception { + Origin groupOrigin = new Origin(); + groupOrigin.setType(OriginType.NATIVE); + + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(mockContext.getActorUrn()).thenReturn(USER_URN_STRING); + when(_groupService.groupExists(any())).thenReturn(true); + when(_groupService.getGroupOrigin(eq(_groupUrn))).thenReturn(groupOrigin); + + _resolver.get(_dataFetchingEnvironment).join(); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java index 38b3b310c1302d..822d353218ab3d 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java @@ -64,7 +64,7 @@ public static DataHubIngestionSourceInfo getTestIngestionSourceInfo() { info.setName("My Test Source"); info.setType("mysql"); info.setSchedule(new DataHubIngestionSourceSchedule().setTimezone("UTC").setInterval("* * * * *")); - info.setConfig(new DataHubIngestionSourceConfig().setVersion("0.8.18").setRecipe("my recipe").setExecutorId("executor id")); + info.setConfig(new DataHubIngestionSourceConfig().setVersion("0.8.18").setRecipe("{}").setExecutorId("executor id")); return info; } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java new file mode 100644 index 00000000000000..c855f974d32bf0 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateTestConnectionRequestResolverTest.java @@ -0,0 +1,65 @@ +package com.linkedin.datahub.graphql.resolvers.ingest.execution; + +import com.datahub.authentication.Authentication; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateTestConnectionRequestInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.config.IngestionConfiguration; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; +import static org.testng.Assert.*; + + +public class CreateTestConnectionRequestResolverTest { + + private static final CreateTestConnectionRequestInput TEST_INPUT = new CreateTestConnectionRequestInput( + "test recipe" + ); + + @Test + public void testGetSuccess() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + IngestionConfiguration ingestionConfiguration = new IngestionConfiguration(); + ingestionConfiguration.setDefaultCliVersion("default"); + CreateTestConnectionRequestResolver resolver = new CreateTestConnectionRequestResolver(mockClient, ingestionConfiguration); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + IngestionConfiguration ingestionConfiguration = new IngestionConfiguration(); + ingestionConfiguration.setDefaultCliVersion("default"); + CreateTestConnectionRequestResolver resolver = new CreateTestConnectionRequestResolver(mockClient, ingestionConfiguration); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(RuntimeException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(Authentication.class)); + } +} + diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java new file mode 100644 index 00000000000000..04ed7720333151 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java @@ -0,0 +1,174 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.Deprecation; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchDatasetUpdateInput; +import com.linkedin.datahub.graphql.generated.Dataset; +import com.linkedin.datahub.graphql.generated.DatasetDeprecationUpdate; +import com.linkedin.datahub.graphql.generated.DatasetUpdateInput; +import com.linkedin.datahub.graphql.types.BatchMutableType; +import com.linkedin.datahub.graphql.types.dataset.DatasetType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.testng.annotations.Test; +import com.linkedin.entity.Aspect; + +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.CompletionException; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + +public class MutableTypeBatchResolverTest { + + private static final String TEST_DATASET_1_URN = "urn:li:dataset:id-1"; + private static final String TEST_DATASET_2_URN = "urn:li:dataset:id-2"; + private static final boolean TEST_DATASET_1_IS_DEPRECATED = true; + private static final boolean TEST_DATASET_2_IS_DEPRECATED = false; + private static final String TEST_DATASET_1_DEPRECATION_NOTE = "Test Deprecation Note"; + private static final String TEST_DATASET_2_DEPRECATION_NOTE = ""; + private static final Deprecation TEST_DATASET_1_DEPRECATION; + + static { + try { + TEST_DATASET_1_DEPRECATION = new Deprecation() + .setDeprecated(TEST_DATASET_1_IS_DEPRECATED) + .setNote(TEST_DATASET_1_DEPRECATION_NOTE) + .setActor(Urn.createFromString("urn:li:corpuser:datahub")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private static final Deprecation TEST_DATASET_2_DEPRECATION; + + static { + try { + TEST_DATASET_2_DEPRECATION = new Deprecation() + .setDeprecated(TEST_DATASET_2_IS_DEPRECATED) + .setNote(TEST_DATASET_2_DEPRECATION_NOTE) + .setActor(Urn.createFromString("urn:li:corpuser:datahub")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockClient = Mockito.mock(RestliEntityClient.class); + BatchMutableType batchMutableType = new DatasetType(mockClient); + + MutableTypeBatchResolver resolver = new MutableTypeBatchResolver<>(batchMutableType); + + List mockInputs = Arrays.asList( + new BatchDatasetUpdateInput.Builder() + .setUrn(TEST_DATASET_1_URN) + .setUpdate( + new DatasetUpdateInput.Builder() + .setDeprecation( + new DatasetDeprecationUpdate.Builder() + .setDeprecated(TEST_DATASET_1_IS_DEPRECATED) + .setNote(TEST_DATASET_1_DEPRECATION_NOTE) + .build() + ) + .build() + ) + .build(), + new BatchDatasetUpdateInput.Builder() + .setUrn(TEST_DATASET_2_URN) + .setUpdate( + new DatasetUpdateInput.Builder() + .setDeprecation( + new DatasetDeprecationUpdate.Builder() + .setDeprecated(TEST_DATASET_2_IS_DEPRECATED) + .setNote(TEST_DATASET_2_DEPRECATION_NOTE) + .build() + ) + .build() + ) + .build() + ); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(mockInputs); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Authentication mockAuth = Mockito.mock(Authentication.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuth); + Mockito.when(mockAuth.getActor()).thenReturn(new Actor(ActorType.USER, "datahub")); + + Urn datasetUrn1 = Urn.createFromString(TEST_DATASET_1_URN); + Urn datasetUrn2 = Urn.createFromString(TEST_DATASET_2_URN); + + Mockito.when(mockClient.batchGetV2(Mockito.eq(Constants.DATASET_ENTITY_NAME), + Mockito.eq(new HashSet<>(ImmutableSet.of(datasetUrn1, datasetUrn2))), + Mockito.any(), + Mockito.any(Authentication.class))) + .thenReturn(ImmutableMap.of( + datasetUrn1, + new EntityResponse() + .setEntityName(Constants.DATASET_ENTITY_NAME) + .setUrn(datasetUrn1) + .setAspects(new EnvelopedAspectMap(ImmutableMap.of( + Constants.DATASET_DEPRECATION_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_DATASET_1_DEPRECATION.data())) + ))), + datasetUrn2, + new EntityResponse() + .setEntityName(Constants.DATASET_ENTITY_NAME) + .setUrn(datasetUrn2) + .setAspects(new EnvelopedAspectMap(ImmutableMap.of( + Constants.DATASET_DEPRECATION_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_DATASET_2_DEPRECATION.data())) + ))) + )); + + List result = resolver.get(mockEnv).join(); + + ArgumentCaptor> changeProposalCaptor = ArgumentCaptor.forClass((Class) Collection.class); + Mockito.verify(mockClient, Mockito.times(1)).batchIngestProposals(changeProposalCaptor.capture(), Mockito.any()); + Mockito.verify(mockClient, Mockito.times(1)).batchGetV2( + Mockito.eq(Constants.DATASET_ENTITY_NAME), + Mockito.eq(ImmutableSet.of(datasetUrn1, datasetUrn2)), + // Dataset aspects to fetch are private, but aren't important for this test + Mockito.any(), + Mockito.any(Authentication.class) + ); + Collection changeProposals = changeProposalCaptor.getValue(); + + assertEquals(changeProposals.size(), 2); + assertEquals(result.size(), 2); + } + + @Test + public void testGetFailureUnauthorized() throws Exception { + EntityClient mockClient = Mockito.mock(RestliEntityClient.class); + BatchMutableType batchMutableType = new DatasetType(mockClient); + + MutableTypeBatchResolver resolver = new MutableTypeBatchResolver<>(batchMutableType); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java new file mode 100644 index 00000000000000..43121fa592fc92 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchAddOwnersResolverTest.java @@ -0,0 +1,290 @@ +package com.linkedin.datahub.graphql.resolvers.owner; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Owner; +import com.linkedin.common.OwnerArray; +import com.linkedin.common.Ownership; +import com.linkedin.common.OwnershipType; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchAddOwnersInput; +import com.linkedin.datahub.graphql.generated.OwnerEntityType; +import com.linkedin.datahub.graphql.generated.OwnerInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddOwnersResolver; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchAddOwnersResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_OWNER_URN_1 = "urn:li:corpuser:test-id-1"; + private static final String TEST_OWNER_URN_2 = "urn:li:corpuser:test-id-2"; + + @Test + public void testGetSuccessNoExistingOwners() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_2))).thenReturn(true); + + BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddOwnersInput input = new BatchAddOwnersInput(ImmutableList.of(new OwnerInput( + TEST_OWNER_URN_1, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER), + new OwnerInput( + TEST_OWNER_URN_2, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(), // Ownership has a dynamically generated timestamp + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_OWNER_URN_1)) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_OWNER_URN_2)) + ); + } + + @Test + public void testGetSuccessExistingOwners() throws Exception { + final Ownership originalOwnership = new Ownership().setOwners(new OwnerArray(ImmutableList.of( + new Owner().setOwner(Urn.createFromString(TEST_OWNER_URN_1)).setType(OwnershipType.TECHNICAL_OWNER) + ))); + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalOwnership); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalOwnership); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_2))).thenReturn(true); + + BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddOwnersInput input = new BatchAddOwnersInput(ImmutableList.of(new OwnerInput( + TEST_OWNER_URN_1, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER), + new OwnerInput( + TEST_OWNER_URN_2, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(), // Ownership has a dynamically generated timestamp + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_OWNER_URN_1)) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_OWNER_URN_2)) + ); + } + + @Test + public void testGetFailureOwnerDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(false); + + BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddOwnersInput input = new BatchAddOwnersInput(ImmutableList.of(new OwnerInput( + TEST_OWNER_URN_1, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER), + new OwnerInput( + TEST_OWNER_URN_2, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(true); + + BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddOwnersInput input = new BatchAddOwnersInput(ImmutableList.of(new OwnerInput( + TEST_OWNER_URN_1, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER), + new OwnerInput( + TEST_OWNER_URN_2, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddOwnersInput input = new BatchAddOwnersInput(ImmutableList.of(new OwnerInput( + TEST_OWNER_URN_1, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER), + new OwnerInput( + TEST_OWNER_URN_2, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchAddOwnersResolver resolver = new BatchAddOwnersResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchAddOwnersInput input = new BatchAddOwnersInput(ImmutableList.of(new OwnerInput( + TEST_OWNER_URN_1, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER), + new OwnerInput( + TEST_OWNER_URN_2, + OwnerEntityType.CORP_USER, + com.linkedin.datahub.graphql.generated.OwnershipType.BUSINESS_OWNER)), + ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java new file mode 100644 index 00000000000000..ac4e0a7cdbef63 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/owner/BatchRemoveOwnersResolverTest.java @@ -0,0 +1,206 @@ +package com.linkedin.datahub.graphql.resolvers.owner; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Owner; +import com.linkedin.common.OwnerArray; +import com.linkedin.common.Ownership; +import com.linkedin.common.OwnershipType; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchRemoveOwnersInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveOwnersResolver; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchRemoveOwnersResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_OWNER_URN_1 = "urn:li:corpuser:test-id-1"; + private static final String TEST_OWNER_URN_2 = "urn:li:corpuser:test-id-2"; + + @Test + public void testGetSuccessNoExistingOwners() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_2))).thenReturn(true); + + BatchRemoveOwnersResolver resolver = new BatchRemoveOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveOwnersInput input = new BatchRemoveOwnersInput(ImmutableList.of( + TEST_OWNER_URN_1, + TEST_OWNER_URN_2 + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(), // Ownership has a dynamically generated timestamp + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetSuccessExistingOwners() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + final Ownership oldOwners1 = new Ownership().setOwners(new OwnerArray(ImmutableList.of( + new Owner().setOwner(Urn.createFromString(TEST_OWNER_URN_1)).setType(OwnershipType.TECHNICAL_OWNER) + ))); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(oldOwners1); + + final Ownership oldOwners2 = new Ownership().setOwners(new OwnerArray(ImmutableList.of( + new Owner().setOwner(Urn.createFromString(TEST_OWNER_URN_2)).setType(OwnershipType.TECHNICAL_OWNER) + ))); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(oldOwners2); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_2))).thenReturn(true); + + BatchRemoveOwnersResolver resolver = new BatchRemoveOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveOwnersInput input = new BatchRemoveOwnersInput(ImmutableList.of(TEST_OWNER_URN_1, TEST_OWNER_URN_2 + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.OWNERSHIP_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_OWNER_URN_1))).thenReturn(true); + + BatchRemoveOwnersResolver resolver = new BatchRemoveOwnersResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveOwnersInput input = new BatchRemoveOwnersInput(ImmutableList.of(TEST_OWNER_URN_1, TEST_OWNER_URN_2 + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchRemoveOwnersResolver resolver = new BatchRemoveOwnersResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveOwnersInput input = new BatchRemoveOwnersInput(ImmutableList.of(TEST_OWNER_URN_1, TEST_OWNER_URN_2 + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchRemoveOwnersResolver resolver = new BatchRemoveOwnersResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchRemoveOwnersInput input = new BatchRemoveOwnersInput(ImmutableList.of(TEST_OWNER_URN_1, TEST_OWNER_URN_2 + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java new file mode 100644 index 00000000000000..0eb361138002d3 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchAddTagsResolverTest.java @@ -0,0 +1,308 @@ +package com.linkedin.datahub.graphql.resolvers.tag; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchAddTagsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddTagsResolver; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchAddTagsResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_TAG_1_URN = "urn:li:tag:test-id-1"; + private static final String TEST_TAG_2_URN = "urn:li:tag:test-id-2"; + + @Test + public void testGetSuccessNoExistingTags() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_2_URN))).thenReturn(true); + + BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTagsInput input = new BatchAddTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final GlobalTags newTags = new GlobalTags().setTags(new TagAssociationArray(ImmutableList.of( + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN)), + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) + ))); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newTags)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newTags)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TAG_1_URN)) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TAG_2_URN)) + ); + } + + @Test + public void testGetSuccessExistingTags() throws Exception { + GlobalTags originalTags = new GlobalTags().setTags(new TagAssociationArray(ImmutableList.of( + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN)))) + ); + + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalTags); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalTags); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_2_URN))).thenReturn(true); + + BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTagsInput input = new BatchAddTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final GlobalTags newTags = new GlobalTags().setTags(new TagAssociationArray(ImmutableList.of( + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN)), + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) + ))); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(newTags)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(newTags)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TAG_1_URN)) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_TAG_2_URN)) + ); + } + + @Test + public void testGetFailureTagDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(false); + + BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTagsInput input = new BatchAddTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(true); + + BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTagsInput input = new BatchAddTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTagsInput input = new BatchAddTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchAddTagsResolver resolver = new BatchAddTagsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchAddTagsInput input = new BatchAddTagsInput(ImmutableList.of( + TEST_TAG_1_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java new file mode 100644 index 00000000000000..124927ff0ae7bc --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/BatchRemoveTagsResolverTest.java @@ -0,0 +1,260 @@ +package com.linkedin.datahub.graphql.resolvers.tag; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchRemoveTagsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveTagsResolver; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchRemoveTagsResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_TAG_1_URN = "urn:li:tag:test-id-1"; + private static final String TEST_TAG_2_URN = "urn:li:tag:test-id-2"; + + @Test + public void testGetSuccessNoExistingTags() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_2_URN))).thenReturn(true); + + BatchRemoveTagsResolver resolver = new BatchRemoveTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTagsInput input = new BatchRemoveTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final GlobalTags emptyTags = new GlobalTags().setTags(new TagAssociationArray(Collections.emptyList())); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetSuccessExistingTags() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + final GlobalTags oldTags1 = new GlobalTags().setTags(new TagAssociationArray(ImmutableList.of( + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN)), + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_2_URN)) + ))); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(oldTags1); + + final GlobalTags oldTags2 = new GlobalTags().setTags(new TagAssociationArray(ImmutableList.of( + new TagAssociation().setTag(TagUrn.createFromString(TEST_TAG_1_URN)) + ))); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(oldTags2); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_2_URN))).thenReturn(true); + + BatchRemoveTagsResolver resolver = new BatchRemoveTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTagsInput input = new BatchRemoveTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + final GlobalTags emptyTags = new GlobalTags().setTags(new TagAssociationArray(Collections.emptyList())); + + final MetadataChangeProposal proposal1 = new MetadataChangeProposal(); + proposal1.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_1)); + proposal1.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal1.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal1.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); + proposal1.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal1), + Mockito.any(AuditStamp.class) + ); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString(TEST_ENTITY_URN_2)); + proposal2.setEntityType(Constants.DATASET_ENTITY_NAME); + proposal2.setAspectName(Constants.GLOBAL_TAGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(emptyTags)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOBAL_TAGS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TAG_1_URN))).thenReturn(true); + + BatchRemoveTagsResolver resolver = new BatchRemoveTagsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTagsInput input = new BatchRemoveTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchRemoveTagsResolver resolver = new BatchRemoveTagsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTagsInput input = new BatchRemoveTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchRemoveTagsResolver resolver = new BatchRemoveTagsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchRemoveTagsInput input = new BatchRemoveTagsInput(ImmutableList.of( + TEST_TAG_1_URN, + TEST_TAG_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java new file mode 100644 index 00000000000000..3079cfc074caa5 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/CreateTagResolverTest.java @@ -0,0 +1,100 @@ +package com.linkedin.datahub.graphql.resolvers.tag; + +import com.datahub.authentication.Authentication; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateTagInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.tag.TagProperties; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.key.TagKey; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class CreateTagResolverTest { + + private static final CreateTagInput TEST_INPUT = new CreateTagInput( + "test-id", + "test-name", + "test-description" + ); + + @Test + public void testGetSuccess() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.ingestProposal(Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class))) + .thenReturn(String.format("urn:li:tag:%s", TEST_INPUT.getId())); + CreateTagResolver resolver = new CreateTagResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + final TagKey key = new TagKey(); + key.setName("test-id"); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + proposal.setEntityType(Constants.TAG_ENTITY_NAME); + TagProperties props = new TagProperties(); + props.setDescription("test-description"); + props.setName("test-name"); + proposal.setAspectName(Constants.TAG_PROPERTIES_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(props)); + proposal.setChangeType(ChangeType.UPSERT); + + // Not ideal to match against "any", but we don't know the auto-generated execution request id + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + CreateTagResolver resolver = new CreateTagResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.doThrow(RuntimeException.class).when(mockClient).ingestProposal( + Mockito.any(), + Mockito.any(Authentication.class)); + CreateTagResolver resolver = new CreateTagResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/DeleteTagResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/DeleteTagResolverTest.java new file mode 100644 index 00000000000000..b01ac1a9b14ae9 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/DeleteTagResolverTest.java @@ -0,0 +1,56 @@ +package com.linkedin.datahub.graphql.resolvers.tag; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class DeleteTagResolverTest { + + private static final String TEST_URN = "urn:li:tag:test-id"; + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + DeleteTagResolver resolver = new DeleteTagResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockClient, Mockito.times(1)).deleteEntity( + Mockito.eq(Urn.createFromString(TEST_URN)), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + DeleteTagResolver resolver = new DeleteTagResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).deleteEntity( + Mockito.any(), + Mockito.any(Authentication.class)); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java new file mode 100644 index 00000000000000..78655daf13776d --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchAddTermsResolverTest.java @@ -0,0 +1,252 @@ +package com.linkedin.datahub.graphql.resolvers.term; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlossaryTermAssociation; +import com.linkedin.common.GlossaryTermAssociationArray; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.urn.GlossaryTermUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchAddTermsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchAddTermsResolver; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchAddTermsResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_GLOSSARY_TERM_1_URN = "urn:li:glossaryTerm:test-id-1"; + private static final String TEST_GLOSSARY_TERM_2_URN = "urn:li:glossaryTerm:test-id-2"; + + @Test + public void testGetSuccessNoExistingTerms() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_GLOSSARY_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_GLOSSARY_TERM_2_URN))).thenReturn(true); + + BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTermsInput input = new BatchAddTermsInput(ImmutableList.of(TEST_GLOSSARY_TERM_1_URN, + TEST_GLOSSARY_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), // glossary terms contains a dynamically generated audit stamp + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_GLOSSARY_TERM_1_URN)) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_GLOSSARY_TERM_2_URN)) + ); + } + + @Test + public void testGetSuccessExistingTerms() throws Exception { + GlossaryTerms originalTerms = new GlossaryTerms().setTerms(new GlossaryTermAssociationArray(ImmutableList.of( + new GlossaryTermAssociation().setUrn(GlossaryTermUrn.createFromString(TEST_GLOSSARY_TERM_1_URN)))) + ); + + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalTerms); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(originalTerms); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_GLOSSARY_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_GLOSSARY_TERM_2_URN))).thenReturn(true); + + BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTermsInput input = new BatchAddTermsInput(ImmutableList.of( + TEST_GLOSSARY_TERM_1_URN, + TEST_GLOSSARY_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), // glossary terms contains a dynamically generated audit stamp + Mockito.any(AuditStamp.class) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_GLOSSARY_TERM_1_URN)) + ); + + Mockito.verify(mockService, Mockito.times(1)).exists( + Mockito.eq(Urn.createFromString(TEST_GLOSSARY_TERM_2_URN)) + ); + } + + @Test + public void testGetFailureTagDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_GLOSSARY_TERM_1_URN))).thenReturn(false); + + BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTermsInput input = new BatchAddTermsInput(ImmutableList.of(TEST_GLOSSARY_TERM_1_URN, + TEST_GLOSSARY_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_GLOSSARY_TERM_1_URN))).thenReturn(true); + + BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTermsInput input = new BatchAddTermsInput(ImmutableList.of(TEST_GLOSSARY_TERM_1_URN, + TEST_GLOSSARY_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchAddTermsInput input = new BatchAddTermsInput(ImmutableList.of(TEST_GLOSSARY_TERM_1_URN, + TEST_GLOSSARY_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchAddTermsResolver resolver = new BatchAddTermsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchAddTermsInput input = new BatchAddTermsInput(ImmutableList.of(TEST_GLOSSARY_TERM_1_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java new file mode 100644 index 00000000000000..cc5d825ac5ee56 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/term/BatchRemoveTermsResolverTest.java @@ -0,0 +1,215 @@ +package com.linkedin.datahub.graphql.resolvers.term; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlossaryTermAssociation; +import com.linkedin.common.GlossaryTermAssociationArray; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.urn.GlossaryTermUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchRemoveTermsInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.resolvers.mutate.BatchRemoveTermsResolver; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class BatchRemoveTermsResolverTest { + + private static final String TEST_ENTITY_URN_1 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; + private static final String TEST_ENTITY_URN_2 = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test-2,PROD)"; + private static final String TEST_TERM_1_URN = "urn:li:glossaryTerm:test-id-1"; + private static final String TEST_TERM_2_URN = "urn:li:glossaryTerm:test-id-2"; + + @Test + public void testGetSuccessNoExistingTerms() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_2_URN))).thenReturn(true); + + BatchRemoveTermsResolver resolver = new BatchRemoveTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTermsInput input = new BatchRemoveTermsInput(ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), // Glossary terms contains dynamically generated audit stamp + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetSuccessExistingTerms() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + final GlossaryTerms oldTerms1 = new GlossaryTerms().setTerms(new GlossaryTermAssociationArray(ImmutableList.of( + new GlossaryTermAssociation().setUrn(GlossaryTermUrn.createFromString(TEST_TERM_1_URN)), + new GlossaryTermAssociation().setUrn(GlossaryTermUrn.createFromString(TEST_TERM_2_URN)) + ))); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(oldTerms1); + + final GlossaryTerms oldTerms2 = new GlossaryTerms().setTerms(new GlossaryTermAssociationArray(ImmutableList.of( + new GlossaryTermAssociation().setUrn(GlossaryTermUrn.createFromString(TEST_TERM_1_URN)) + ))); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(oldTerms2); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_2_URN))).thenReturn(true); + + BatchRemoveTermsResolver resolver = new BatchRemoveTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTermsInput input = new BatchRemoveTermsInput(ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), // Glossary terms contains dynamically generated audit stamp + Mockito.any(AuditStamp.class) + ); + } + + @Test + public void testGetFailureResourceDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_1)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + Mockito.when(mockService.getAspect( + Mockito.eq(UrnUtils.getUrn(TEST_ENTITY_URN_2)), + Mockito.eq(Constants.GLOSSARY_TERMS_ASPECT_NAME), + Mockito.eq(0L))) + .thenReturn(null); + + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_1))).thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(TEST_ENTITY_URN_2))).thenReturn(true); + Mockito.when(mockService.exists(Urn.createFromString(TEST_TERM_1_URN))).thenReturn(true); + + BatchRemoveTermsResolver resolver = new BatchRemoveTermsResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTermsInput input = new BatchRemoveTermsInput(ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + BatchRemoveTermsResolver resolver = new BatchRemoveTermsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + BatchRemoveTermsInput input = new BatchRemoveTermsInput(ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockService, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + + Mockito.doThrow(RuntimeException.class).when(mockService).ingestProposal( + Mockito.any(), + Mockito.any(AuditStamp.class)); + + BatchRemoveTermsResolver resolver = new BatchRemoveTermsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + BatchRemoveTermsInput input = new BatchRemoveTermsInput(ImmutableList.of( + TEST_TERM_1_URN, + TEST_TERM_2_URN + ), ImmutableList.of( + new ResourceRefInput(TEST_ENTITY_URN_1, null, null), + new ResourceRefInput(TEST_ENTITY_URN_2, null, null))); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserInviteTokenResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserInviteTokenResolverTest.java new file mode 100644 index 00000000000000..cf66367272fe59 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserInviteTokenResolverTest.java @@ -0,0 +1,50 @@ +package com.linkedin.datahub.graphql.resolvers.user; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.datahub.graphql.QueryContext; +import graphql.schema.DataFetchingEnvironment; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class CreateNativeUserInviteTokenResolverTest { + + private static final String INVITE_TOKEN = "inviteToken"; + + private NativeUserService _nativeUserService; + private CreateNativeUserInviteTokenResolver _resolver; + private DataFetchingEnvironment _dataFetchingEnvironment; + private Authentication _authentication; + + @BeforeMethod + public void setupTest() { + _nativeUserService = mock(NativeUserService.class); + _dataFetchingEnvironment = mock(DataFetchingEnvironment.class); + _authentication = mock(Authentication.class); + + _resolver = new CreateNativeUserInviteTokenResolver(_nativeUserService); + } + + @Test + public void testFailsCannotManageUserCredentials() { + QueryContext mockContext = getMockDenyContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testPasses() throws Exception { + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(_nativeUserService.generateNativeUserInviteToken(any())).thenReturn(INVITE_TOKEN); + + assertEquals(INVITE_TOKEN, _resolver.get(_dataFetchingEnvironment).join().getInviteToken()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserResetTokenResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserResetTokenResolverTest.java new file mode 100644 index 00000000000000..2164d4160634ce --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/CreateNativeUserResetTokenResolverTest.java @@ -0,0 +1,66 @@ +package com.linkedin.datahub.graphql.resolvers.user; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateNativeUserResetTokenInput; +import graphql.schema.DataFetchingEnvironment; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class CreateNativeUserResetTokenResolverTest { + + private static final String RESET_TOKEN = "resetToken"; + private static final String USER_URN_STRING = "urn:li:corpuser:test"; + + private NativeUserService _nativeUserService; + private CreateNativeUserResetTokenResolver _resolver; + private DataFetchingEnvironment _dataFetchingEnvironment; + private Authentication _authentication; + + @BeforeMethod + public void setupTest() { + _nativeUserService = mock(NativeUserService.class); + _dataFetchingEnvironment = mock(DataFetchingEnvironment.class); + _authentication = mock(Authentication.class); + + _resolver = new CreateNativeUserResetTokenResolver(_nativeUserService); + } + + @Test + public void testFailsCannotManageUserCredentials() { + QueryContext mockContext = getMockDenyContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testFailsNullUserUrn() throws Exception { + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + CreateNativeUserResetTokenInput input = new CreateNativeUserResetTokenInput(null); + when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(_nativeUserService.generateNativeUserPasswordResetToken(any(), any())).thenReturn(RESET_TOKEN); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testPasses() throws Exception { + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + CreateNativeUserResetTokenInput input = new CreateNativeUserResetTokenInput(USER_URN_STRING); + when(_dataFetchingEnvironment.getArgument(eq("input"))).thenReturn(input); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(_nativeUserService.generateNativeUserPasswordResetToken(any(), any())).thenReturn(RESET_TOKEN); + + assertEquals(RESET_TOKEN, _resolver.get(_dataFetchingEnvironment).join().getResetToken()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/GetNativeUserInviteTokenResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/GetNativeUserInviteTokenResolverTest.java new file mode 100644 index 00000000000000..0870bcae5661b4 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/user/GetNativeUserInviteTokenResolverTest.java @@ -0,0 +1,50 @@ +package com.linkedin.datahub.graphql.resolvers.user; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.datahub.graphql.QueryContext; +import graphql.schema.DataFetchingEnvironment; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class GetNativeUserInviteTokenResolverTest { + + private static final String INVITE_TOKEN = "inviteToken"; + + private NativeUserService _nativeUserService; + private GetNativeUserInviteTokenResolver _resolver; + private DataFetchingEnvironment _dataFetchingEnvironment; + private Authentication _authentication; + + @BeforeMethod + public void setupTest() { + _nativeUserService = mock(NativeUserService.class); + _dataFetchingEnvironment = mock(DataFetchingEnvironment.class); + _authentication = mock(Authentication.class); + + _resolver = new GetNativeUserInviteTokenResolver(_nativeUserService); + } + + @Test + public void testFailsCannotManageUserCredentials() { + QueryContext mockContext = getMockDenyContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + + assertThrows(() -> _resolver.get(_dataFetchingEnvironment).join()); + } + + @Test + public void testPasses() throws Exception { + QueryContext mockContext = getMockAllowContext(); + when(_dataFetchingEnvironment.getContext()).thenReturn(mockContext); + when(mockContext.getAuthentication()).thenReturn(_authentication); + when(_nativeUserService.getNativeUserInviteToken(any())).thenReturn(INVITE_TOKEN); + + assertEquals(INVITE_TOKEN, _resolver.get(_dataFetchingEnvironment).join().getInviteToken()); + } +} diff --git a/datahub-ranger-plugin/README.md b/datahub-ranger-plugin/README.md new file mode 100644 index 00000000000000..a857a4955fe6f7 --- /dev/null +++ b/datahub-ranger-plugin/README.md @@ -0,0 +1,266 @@ +--- +title: "Configuring Authorization with Apache Ranger" +hide_title: true +--- +# Configuring Authorization with Apache Ranger +DataHub integration with Apache Ranger allows DataHub Authorization policies to be controlled inside Apache Ranger. +Admins can create users, groups and roles on Apache Ranger, and then assign them to Ranger policies to control the authorization of requests to DataHub. + +We'll break down configuration of the DataHub Apache Ranger Plugin into two parts: + +1. Configuring your Apache Ranger Deployment +2. Configuring your DataHub Deployment + +> Disclaimer: All configurations shown in this documented were tested against [Privacera Platform](https://privacera.com/) v6.3.0.1. + +# Prerequisites +- User identifier present in CorpRole URN should be the name of the AD/LDAP user. For example in URN **urn:li:corpuser:datahub**, the **datahub** should present as name of user in AD/LDAP +- Apache Ranger and DataHub are configured for authentication via same IDP (either LDAP + JaaS or OIDC SSO) +- Apache Ranger service available via HTTP +- Basic authentication is enabled on Apache Ranger Service + +# Configuration + +## Configuring your Apache Ranger Deployment + +Perform the following steps to configure an Apache Ranger deployment to support creating access policies compatible with DataHub. +For kubernetes example command, please replace the <ranger-pod-name> and <namespace> as per your environment. + +1. Download the **datahub-ranger-plugin** from [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-ranger-plugin) +2. Create a "datahub" directory inside the "ranger-plugins" directory where Apache Ranger is deployed. For example, to do this in a Privacera container + + *Docker command:* + ```bash + docker exec privacera_ranger_1 mkdir ews/webapp/WEB-INF/classes/ranger-plugins/datahub + ``` + *Kubernetes command:* + ```bash + kubectl exec mkdir ews/webapp/WEB-INF/classes/ranger-plugins/datahub -n + ``` +3. Copy the downloaded **datahub-ranger-plugin** jar into the newly created "datahub" directory. For example, to do this in a Privacera container + + *Docker command:* + ```bash + docker cp datahub-ranger-plugin-.jar privacera_ranger_1:/opt/ranger/ranger-2.1.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/datahub/ + ``` + *Kubernetes command:* + ```bash + kubectl cp datahub-ranger-plugin-.jar :/opt/ranger/ranger-2.1.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/datahub/ -n + ``` +4. Download the [service definition file](../datahub-ranger-plugin/conf/servicedef.json). This service definition is the ranger service definition JSON file for datahub-ranger-plugin-<version>.jar +5. Register the downloaded service definition file with Apache Ranger Service. To do this executes the below curl command
+Replace variables with corresponding values in curl command + - <ranger-admin-username> + - <ranger-admin-password> + - <ranger-host> + ```bash + curl -u : -X POST -H "Accept: application/json" -H "Content-Type: application/json" --data @servicedef.json http://:6080/service/public/v2/api/servicedef + ``` + +### Defining a Ranger Policy + +Now, you should have the DataHub plugin registered with Apache Ranger. Next, we'll create a sample user and add them to our first resource policy. + +1. Login into the Apache Ranger UI (Privacera Portal) to performs below steps. +2. Verify **datahub-ranger-plugin** is registered successfully: The **datahub-ranger-plugin** should be visible as **DATAHUB** in *Access Management -> Resource Policies*. +3. Create a service under the plugin **DATAHUB** with name **ranger_datahub** + + **DATAHUB** plugin and **ranger_datahub** service is shown in below screenshot:
+ + ![Privacera Portal DATAHUB screenshot](./doc-images/datahub-plugin.png) + +4. Create a new policy under service **ranger_datahub** - this will be used to control DataHub authorization. +5. Create a test user & assign them to a policy. We'll use the `datahub` user, which is the default root user inside DataHub. + + To do this performs below steps + - Create a user **datahub** + - Create a policy under **ranger_datahub** service. To assign [Platform Privileges](../docs/authorization/policies.md) (e.g. Admin privileges), simply use the "platform" resource type which is defined. To test the flow, we can simply assign the **datahub** user all platform privileges that are available through the Ranger UI. This will enable the "datahub" to have full platform admin privileges. + + > To define fine-grained resource privileges, e.g. for DataHub Datasets, Dashboards, Charts, and more, you can simply select the appropriate Resource Type in the Ranger policy builder. You should also see a list of privileges that are supported for each resource type, which correspond to the actions that you can perform. To learn more about supported privileges, check out the DataHub [Policies Guide](../docs/authorization/policies.md). + + DataHub platform access policy screenshot:
+ + ![Privacera Portal DATAHUB screenshot](./doc-images/datahub-platform-access-policy.png) + +Once we've created our first policy, we can set up DataHub to start authorizing requests using Ranger policies. + + +## Configuring your DataHub Deployment + +Perform the following steps to configure DataHub to send incoming requests to Apache Ranger for authorization. + +1. Download Apache Ranger security xml [ranger-datahub-security.xml](../datahub-ranger-plugin/conf/ranger-datahub-security.xml) +2. In **ranger-datahub-security.xml** edit the value of property *ranger.plugin.datahub.policy.rest.url*. Sample snippet is shown below + ```xml + + ranger.plugin.datahub.policy.rest.url + http://199.209.9.70:6080 + + URL to Ranger Admin + + + ``` + +As per your deployment follow either Docker or Kubernetes section below +### Docker + Configure DataHub to use a Ranger **Authorizer**. On the host where `datahub-gms` is deployed, follow these steps: + 1. Create directory **~/.datahub/plugins/auth/resources/**: Executes below command + ```bash + mkdir -p ~/.datahub/plugins/auth/resources/ + ``` + 2. Copy **ranger-datahub-security.xml** file to ~/.datahub/plugins/auth/resources/ + 3. [Optional] Disable the DataHub default policy authorizer by setting the following environment variable on the `datahub-gms` container: + ```bash + export AUTH_POLICIES_ENABLED=false + ``` + 4. Enable the Apache Ranger authorizer by setting the following environment variable on the `datahub-gms` container: + ```bash + export RANGER_AUTHORIZER_ENABLED=true + ``` + 5. Set the Apache Ranger admin username by setting the following environment variable on the `datahub-gms` container: + ```bash + export RANGER_USERNAME= + ``` + 6. Set the Apache Ranger admin password by setting the following environment variable on the `datahub-gms` container: + ```bash + export RANGER_PASSWORD= + ``` + 7. Redeploy DataHub (`datahub-gms`) with the new environment variables +### Kubernetes + Configure DataHub to use a Ranger **Authorizer**. On the host where `kubectl` is installed, follow these steps: + + For kubernetes example command, please replace the <namespace> as per your environment. + + + 1. Download kubernetes configMap for DataHub Apache Ranger authorizer [auth-plugin-configuration-configMap.kubernetes.yaml](../datahub-ranger-plugin/conf/auth-plugin-configuration-configMap.kubernetes.yaml) + + 2. In **auth-plugin-configuration-configMap.kubernetes.yaml** edit the value of property *ranger.plugin.datahub.policy.rest.url*. Sample snippet is shown below + ```xml + + ranger.plugin.datahub.policy.rest.url + http://199.222.9.70:6080 + + URL to Ranger Admin + + + ``` + + 3. Create a kubernetes configMap resource: Execute below command to create an *auth-plugin-configuration* configMap resource + ```bash + kubectl apply -f auth-plugin-configuration-configMap.kubernetes.yaml -n + ``` + + 4. Edit **datahub-datahub-gms** deployment to set environment variables & volume-mount points: Execute below command to open deployment editor
+ ``` + kubectl edit deployment datahub-datahub-gms + ``` + 1. Add below environment variables in under *spec.template.spec.containers[0].env*: Replace <username> by Apache Ranger admin username and <password> by Apache Ranger admin user password. + ```yaml + - name: AUTH_POLICIES_ENABLED + value: "false" + - name: RANGER_AUTHORIZER_ENABLED + value: "true" + - name: RANGER_USERNAME + value: "" + - name: RANGER_PASSWORD + value: "" + ``` + + 2. Add *volumes* under spec.template.spec: Copy & paste below yaml snippet under *spec.template.spec* + ```yaml + volumes: + - configMap: + name: auth-plugin-configuration + name: auth-resource-volume + ``` + 3. Add *volumeMounts* under spec.template.spec.containers[0]: Copy & paste below yaml snippet under spec.template.spec.containers[0] + ```yaml + volumeMounts: + - mountPath: /etc/datahub/plugins/auth/resources + name: auth-resource-volume + readOnly: true + + ``` + 5. Save and quit the editor + 6. Check status of **datahub-datahub-gms** deployment rollout: Execute below command + ```bash + kubectl rollout status deployment/datahub-datahub-gms + ``` + On successful rollout you should see a message *deployment "datahub-datahub-gms" successfully rolled out* + + +That's it! Now we can test out the integration. + +### Validating your Setup +To verify that things are working as expected, we can test that the root **datahub** user has all Platform Privileges and is able to perform all operations: managing users & groups, creating domains, and more. To do this, simply log into your DataHub deployment via the root DataHub user. + +# Revert the Configuration +If you want to revert your deployment configuration and don't want Apache Ranger to control the authorization of your DataHub deployment +then follow the below sections to undo the configuration steps you have performed in section *Configuring Authorization with Apache Ranger* + +1. Revert Configuration of your Apache Ranger Deployment +2. Revert Configuration of your DataHub Deployment + +## Revert Configuration of your Apache Ranger Deployment + For kubernetes example command, please replace the <ranger-pod-name> and <namespace> as per your environment. + + 1. Delete **ranger_datahub** service: Login into the Privacera Portal and delete service **ranger_datahub** + + **ranger_datahub** service is shown in below screenshot:
+ + ![Privacera Portal DATAHUB screenshot](./doc-images/datahub-plugin.png) + + 2. Delete **datahub** plugin: Execute below curl command to delete **datahub** plugin + Replace variables with corresponding values in curl command + - <ranger-admin-username> + - <ranger-admin-password> + - <ranger-host> + + ```bash + curl -u : -X DELETE -H "Accept: application/json" -H "Content-Type: application/json" http://:6080/service/public/v2/api/servicedef/name/datahub + ``` + 3. Delete **datahub** plugin directory: Execute below command to delete the **datahub** plugin directory from Apache Ranger + + *Docker command:* + ```bash + docker exec privacera_ranger_1 rm -rf ews/webapp/WEB-INF/classes/ranger-plugins/datahub + ``` + *Kubernetes command:* + ```bash + kubectl exec -n -- sh -c 'rm -rf ews/webapp/WEB-INF/classes/ranger-plugins/datahub' + ``` + + +## Revert Configuration of your DataHub Deployment +### Docker + 1. Unset environment variables: Execute below command to unset the environment variables + ```bash + unset AUTH_POLICIES_ENABLED + unset RANGER_AUTHORIZER_ENABLED + unset RANGER_USERNAME + unset RANGER_PASSWORD + ``` + 2. Redeploy DataHub (`datahub-gms`) +### Kubernetes + For kubernetes example command, please replace the <namespace> as per your environment. +1. Open deployment editor: Execute below command + ```bash + kubectl edit deployment datahub-datahub-gms -n + ``` +2. Remove below environments variables + 1. AUTH_POLICIES_ENABLED + 2. RANGER_AUTHORIZER_ENABLED + 3. RANGER_USERNAME + 4. RANGER_PASSWORD +3. Remove below volumes related settings + 1. volumes + 2. volumeMounts +4. Save and quit the editor and use below command to check status of **datahub-datahub-gms** deployment rollout + ```bash + kubectl rollout status deployment/datahub-datahub-gms -n + ``` + On successful rollout you should see a message *deployment "datahub-datahub-gms" successfully rolled out* + + +### Validating your Setup +To verify that things are working as expected, we can test that the root **datahub** user has all Platform Privileges and is able to perform all operations: managing users & groups, creating domains, and more. To do this, simply log into your DataHub deployment via the root DataHub user. diff --git a/datahub-ranger-plugin/build.gradle b/datahub-ranger-plugin/build.gradle new file mode 100644 index 00000000000000..a351856f9fb0f4 --- /dev/null +++ b/datahub-ranger-plugin/build.gradle @@ -0,0 +1,143 @@ +import org.apache.tools.ant.filters.ReplaceTokens + + +apply plugin: 'java' +apply plugin: 'signing' +apply plugin: 'maven-publish' +apply plugin: 'io.codearte.nexus-staging' + + + +import org.apache.tools.ant.filters.ReplaceTokens + + +repositories { + mavenCentral() +} + +java { + withJavadocJar() + withSourcesJar() +} + +test { + useJUnit() +} + +dependencies { + implementation 'org.apache.ranger:ranger-plugins-common:2.2.0' + compile 'org.apache.logging.log4j:log4j-1.2-api:2.17.1' + + testCompile externalDependency.testng +} + +def detailedVersionString = "0.0.0-unknown-SNAPSHOT" +def snapshotVersion = false +if (project.hasProperty("releaseVersion")) { + version = releaseVersion + detailedVersionString = releaseVersion +} else { + try { + // apply this plugin in a try-catch block so that we can handle cases without .git directory + apply plugin: "com.palantir.git-version" + def details = versionDetails() + detailedVersionString = gitVersion() + version = details.lastTag + version = version.startsWith("v")? version.substring(1): version + def suffix = details.isCleanTag? "": "-SNAPSHOT" + snapshotVersion = ! details.isCleanTag + } + catch (Exception e) { + e.printStackTrace() + // last fall back + version = detailedVersionString + } +} +// trim version if it is of size 4 to size 3 +def versionParts = version.tokenize(".") +if (versionParts.size() > 3) { + // at-least 4 part version + // we check if the 4th part is a .0 in which case we want to create a release + if (versionParts[3] != '0') { + snapshotVersion = true + } + versionParts = versionParts[0..2] + version = versionParts[0..2].join('.') +} + +if (snapshotVersion) { + if (versionParts[versionParts.size()-1].isInteger()) { + version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" + } else { + // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version + version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' + } +} + +processResources { + filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) +} + + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + pom { + name = 'DataHub Apache Ranger Plugin' + group = 'io.acryl' + artifactId = 'datahub-ranger-plugin' + description = 'DataHub Apache Ranger plugin for authorization of DataHub resources' + url = 'https://datahubproject.io' + scm { + connection = 'scm:git:git://github.com/datahub-project/datahub.git' + developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git' + url = 'https://github.com/datahub-project/datahub.git' + } + + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + + developers { + developer { + id = 'datahub' + name = 'DataHub' + email = 'datahub@acryl.io' + } + } + } + } + } + + repositories { + maven { + def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/" + def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/" + def ossrhUsername = System.getenv('RELEASE_USERNAME') + def ossrhPassword = System.getenv('RELEASE_PASSWORD') + credentials { + username ossrhUsername + password ossrhPassword + } + url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + } + } +} + +signing { + def signingKey = findProperty("signingKey") + def signingPassword = System.getenv("SIGNING_PASSWORD") + useInMemoryPgpKeys(signingKey, signingPassword) + sign publishing.publications.mavenJava +} + +// Required to submit jar file to staging repo of maven central +nexusStaging { + serverUrl = "https://s01.oss.sonatype.org/service/local/" //required only for projects registered in Sonatype after 2021-02-24 + username = System.getenv("NEXUS_USERNAME") + password = System.getenv("NEXUS_PASSWORD") +} diff --git a/datahub-ranger-plugin/conf/auth-plugin-configuration-configMap.kubernetes.yaml b/datahub-ranger-plugin/conf/auth-plugin-configuration-configMap.kubernetes.yaml new file mode 100644 index 00000000000000..36644d1120b479 --- /dev/null +++ b/datahub-ranger-plugin/conf/auth-plugin-configuration-configMap.kubernetes.yaml @@ -0,0 +1,79 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: auth-plugin-configuration +data: + ranger-datahub-security.xml: | + + + + ranger.plugin.datahub.policy.rest.url + http://[RANGER_SERVICE_HOST]:[RANGER_SERVICE_PORT] + + URL to Ranger Admin + + + + + ranger.plugin.datahub.service.name + datahub + + Name of the Ranger service containing policies for this datahub instance + + + + + ranger.plugin.datahub.policy.source.impl + org.apache.ranger.admin.client.RangerAdminRESTClient + + Class to retrieve policies from the source + + + + + ranger.plugin.datahub.policy.rest.ssl.config.file + ranger-policymgr-ssl.xml + + Path to the file containing SSL details to contact Ranger Admin + + + + + ranger.plugin.datahub.policy.pollIntervalMs + 30000 + + How often to poll for changes in policies? + + + + + ranger.plugin.datahub.policy.cache.dir + /tmp + + Directory where Ranger policies are cached after successful retrieval from the source + + + + + ranger.plugin.datahub.policy.rest.client.connection.timeoutMs + 120000 + + RangerRestClient Connection Timeout in Milli Seconds + + + + + ranger.plugin.datahub.policy.rest.client.read.timeoutMs + 30000 + + RangerRestClient read Timeout in Milli Seconds + + + + ranger.plugin.datahub.service.name + ranger_datahub + + Name of the Ranger service containing policies for this datahub instance + + + diff --git a/datahub-ranger-plugin/conf/ranger-datahub-audit.xml b/datahub-ranger-plugin/conf/ranger-datahub-audit.xml new file mode 100644 index 00000000000000..e7a94b2d5befa6 --- /dev/null +++ b/datahub-ranger-plugin/conf/ranger-datahub-audit.xml @@ -0,0 +1,79 @@ + + + + + + + xasecure.audit.destination.db + false + + + + xasecure.audit.destination.db.jdbc.driver + com.mysql.jdbc.Driver + + + + xasecure.audit.destination.db.jdbc.url + jdbc:mysql://localhost/ranger_audit + + + + xasecure.audit.destination.db.password + rangerlogger + + + + xasecure.audit.destination.db.user + rangerlogger + + + + xasecure.audit.destination.db.batch.filespool.dir + /tmp/audit/db/spool + + + + + + xasecure.audit.destination.hdfs + false + + + + xasecure.audit.destination.hdfs.dir + hdfs://localhost:8020/ranger/audit + + + + xasecure.audit.destination.hdfs.batch.filespool.dir + /tmp/audit/hdfs/spool + + + + + + xasecure.audit.destination.log4j + true + + + + xasecure.audit.destination.log4j.logger + ranger_audit_logger + + \ No newline at end of file diff --git a/datahub-ranger-plugin/conf/ranger-datahub-security.xml b/datahub-ranger-plugin/conf/ranger-datahub-security.xml new file mode 100644 index 00000000000000..d9947873c3326d --- /dev/null +++ b/datahub-ranger-plugin/conf/ranger-datahub-security.xml @@ -0,0 +1,90 @@ + + + + + + ranger.plugin.datahub.policy.rest.url + http://[RANGER_SERVICE_HOST]:[RANGER_SERVICE_PORT] + + URL to Ranger Admin + + + + + ranger.plugin.datahub.service.name + datahub + + Name of the Ranger service containing policies for this datahub instance + + + + + ranger.plugin.datahub.policy.source.impl + org.apache.ranger.admin.client.RangerAdminRESTClient + + Class to retrieve policies from the source + + + + + ranger.plugin.datahub.policy.rest.ssl.config.file + ranger-policymgr-ssl.xml + + Path to the file containing SSL details to contact Ranger Admin + + + + + ranger.plugin.datahub.policy.pollIntervalMs + 30000 + + How often to poll for changes in policies? + + + + + ranger.plugin.datahub.policy.cache.dir + /tmp + + Directory where Ranger policies are cached after successful retrieval from the source + + + + + ranger.plugin.datahub.policy.rest.client.connection.timeoutMs + 120000 + + RangerRestClient Connection Timeout in Milli Seconds + + + + + ranger.plugin.datahub.policy.rest.client.read.timeoutMs + 30000 + + RangerRestClient read Timeout in Milli Seconds + + + + ranger.plugin.datahub.service.name + ranger_datahub + + Name of the Ranger service containing policies for this datahub instance + + + diff --git a/datahub-ranger-plugin/conf/servicedef.json b/datahub-ranger-plugin/conf/servicedef.json new file mode 100644 index 00000000000000..33bec319ff124e --- /dev/null +++ b/datahub-ranger-plugin/conf/servicedef.json @@ -0,0 +1,443 @@ +{ + "name": "datahub", + "label": "DataHub Ranger Plugin", + "description": "DataHub metadata service ranger plugin", + "guid": "4b4d4245-dd79-45fa-8854-8eb7cb1c37ad", + "implClass": "com.datahub.authorizer.plugin.ranger.DataHubRangerAuthPlugin", + "version": 1, + "isEnabled": 1, + "resources": [ + { + "itemId": 1, + "name": "platform", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "platform", + "description": "This resource is added to simulate DataHub platform policy" + }, + { + "itemId": 2, + "name": "dataset", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "dataset", + "description": "DataHub dataset resource" + }, + { + "itemId": 3, + "name": "dashboard", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "dashboard", + "description": "DataHub dashboard resource" + }, + { + "itemId": 4, + "name": "chart", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "chart", + "description": "DataHub chart resource" + }, + { + "itemId": 5, + "name": "dataflow", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "dataFlow", + "description": "DataHub dataFlow resource" + }, + { + "itemId": 6, + "name": "datajob", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "dataJob", + "description": "DataHub dataJob resource" + }, + { + "itemId": 7, + "name": "tag", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "tag", + "description": "DataHub tag resource" + }, + { + "itemId": 8, + "name": "container", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "container", + "description": "DataHub container resource" + }, + { + "itemId": 9, + "name": "domain", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "domain", + "description": "DataHub domain resource" + }, + { + "itemId": 10, + "name": "glossaryterm", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "glossaryTerm", + "description": "DataHub glossaryTerm resource" + }, + { + "itemId": 11, + "name": "corpgroup", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "corpGroup", + "description": "DataHub corpGroup resource" + }, + { + "itemId": 12, + "name": "corpuser", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "corpUser", + "description": "DataHub corpUser resource" + }, + { + "itemId": 13, + "name": "notebook", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "notebook", + "description": "DataHub notebook resource" + } + + ], + "accessTypes": [ + { + "itemId": 1, + "name": "MANAGE_POLICIES", + "label": "MANAGE_POLICIES" + }, + { + "itemId": 2, + "name": "MANAGE_INGESTION", + "label": "MANAGE_INGESTION" + }, + { + "itemId": 3, + "name": "MANAGE_SECRETS", + "label": "MANAGE_SECRETS" + }, + { + "itemId": 4, + "name": "MANAGE_USERS_AND_GROUPS", + "label": "MANAGE_USERS_AND_GROUPS" + }, + { + "itemId": 5, + "name": "VIEW_ANALYTICS", + "label": "VIEW_ANALYTICS" + }, + { + "itemId": 6, + "name": "GENERATE_PERSONAL_ACCESS_TOKENS", + "label": "GENERATE_PERSONAL_ACCESS_TOKENS" + }, + { + "itemId": 7, + "name": "MANAGE_DOMAINS", + "label": "MANAGE_DOMAINS" + }, + { + "itemId": 8, + "name": "VIEW_ENTITY_PAGE", + "label": "VIEW_ENTITY_PAGE" + }, + { + "itemId": 9, + "name": "EDIT_ENTITY_TAGS", + "label": "EDIT_ENTITY_TAGS" + }, + { + "itemId": 10, + "name": "EDIT_ENTITY_GLOSSARY_TERMS", + "label": "EDIT_ENTITY_GLOSSARY_TERMS" + }, + { + "itemId": 11, + "name": "EDIT_ENTITY_OWNERS", + "label": "EDIT_ENTITY_OWNERS" + }, + { + "itemId": 12, + "name": "EDIT_ENTITY_DOCS", + "label": "EDIT_ENTITY_DOCS" + }, + { + "itemId": 13, + "name": "EDIT_ENTITY_DOC_LINKS", + "label": "EDIT_ENTITY_DOC_LINKS" + }, + { + "itemId": 14, + "name": "EDIT_ENTITY_STATUS", + "label": "EDIT_ENTITY_STATUS" + }, + { + "itemId": 15, + "name": "EDIT_DOMAINS", + "label": "EDIT_DOMAINS" + }, + { + "itemId": 16, + "name": "EDIT_DEPRECATION", + "label": "EDIT_DEPRECATION" + }, + { + "itemId": 17, + "name": "EDIT_ENTITY_ASSERTIONS", + "label": "EDIT_ENTITY_ASSERTIONS" + }, + { + "itemId": 18, + "name": "EDIT_ENTITY", + "label": "EDIT_ENTITY" + }, + { + "itemId": 19, + "name": "EDIT_DATASET_COL_TAGS", + "label": "EDIT_DATASET_COL_TAGS" + }, + { + "itemId": 20, + "name": "EDIT_DATASET_COL_GLOSSARY_TERMS", + "label": "EDIT_DATASET_COL_GLOSSARY_TERMS" + }, + { + "itemId": 21, + "name": "EDIT_DATASET_COL_DESCRIPTION", + "label": "EDIT_DATASET_COL_DESCRIPTION" + }, + { + "itemId": 22, + "name": "VIEW_DATASET_USAGE", + "label": "VIEW_DATASET_USAGE" + }, + { + "itemId": 23, + "name": "VIEW_DATASET_PROFILE", + "label": "VIEW_DATASET_PROFILE" + }, + { + "itemId": 24, + "name": "EDIT_TAG_COLOR", + "label": "EDIT_TAG_COLOR" + }, + { + "itemId": 25, + "name": "EDIT_GROUP_MEMBERS", + "label": "EDIT_GROUP_MEMBERS" + }, + { + "itemId": 26, + "name": "EDIT_USER_PROFILE", + "label": "EDIT_USER_PROFILE" + }, + { + "itemId": 27, + "name": "EDIT_CONTACT_INFO", + "label": "EDIT_CONTACT_INFO" + }, + { + "itemId": 28, + "name": "COMMON_ENTITYS", + "label": "COMMON_ENTITYS" + }, + { + "itemId": 29, + "name": "EDIT_ENTITY_DOMAINS", + "label": "EDIT_ENTITY_DOMAINS" + }, + { + "itemId": 30, + "name": "EDIT_ENTITY_DEPRECATION", + "label": "EDIT_ENTITY_DEPRECATION" + } + ], + "enums": [ + ], + "contextEnrichers": [ + ], + "policyConditions": [ + ] +} diff --git a/datahub-ranger-plugin/doc-images/datahub-platform-access-policy.png b/datahub-ranger-plugin/doc-images/datahub-platform-access-policy.png new file mode 100644 index 00000000000000..7e3ff6fd372a9d Binary files /dev/null and b/datahub-ranger-plugin/doc-images/datahub-platform-access-policy.png differ diff --git a/datahub-ranger-plugin/doc-images/datahub-plugin.png b/datahub-ranger-plugin/doc-images/datahub-plugin.png new file mode 100644 index 00000000000000..5dd044c0146570 Binary files /dev/null and b/datahub-ranger-plugin/doc-images/datahub-plugin.png differ diff --git a/datahub-ranger-plugin/src/main/java/com/datahub/authorizer/plugin/ranger/DataHubRangerAuthPlugin.java b/datahub-ranger-plugin/src/main/java/com/datahub/authorizer/plugin/ranger/DataHubRangerAuthPlugin.java new file mode 100644 index 00000000000000..f339623052ffd0 --- /dev/null +++ b/datahub-ranger-plugin/src/main/java/com/datahub/authorizer/plugin/ranger/DataHubRangerAuthPlugin.java @@ -0,0 +1,56 @@ +package com.datahub.authorizer.plugin.ranger; + +import org.apache.ranger.plugin.client.BaseClient; +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * Datahub Apache Ranger Plugin. + * It assists in creating policies on Apache Ranger Admin Portal. + * + */ +public class DataHubRangerAuthPlugin extends RangerBaseService { + private static Logger log = Logger.getLogger(DataHubRangerAuthPlugin.class.getName()); + + /** + * This is dummy function. As this plugin doesn't have any configuration + * @return A Map with success message + * @throws Exception + */ + @Override + public Map validateConfig() throws Exception { + throw new UnsupportedOperationException("validateConfig is not supported."); + } + + /** + * This is dummy function. As this plugin doesn't support the resource lookup + * @param resourceLookupContext + * @return Empty list of string + * @throws Exception + */ + @Override + public List lookupResource(ResourceLookupContext resourceLookupContext) throws Exception { + throw new UnsupportedOperationException("lookupResource is not supported."); + } + + private Map returnSuccessMap() { + String message = "Connection test successful"; + Map retMap = new HashMap<>(); + BaseClient.generateResponseDataMap(true, message, message, null, null, retMap); + return retMap; + } + + private Map returnFailMap() { + String message = "Connection test fail"; + Map retMap = new HashMap<>(); + BaseClient.generateResponseDataMap(false, message, message, null, null, retMap); + return retMap; + } + +} diff --git a/datahub-ranger-plugin/src/test/java/com/datahub/authorizer/plugin/ranger/TestDataHubRangerAuthPlugin.java b/datahub-ranger-plugin/src/test/java/com/datahub/authorizer/plugin/ranger/TestDataHubRangerAuthPlugin.java new file mode 100644 index 00000000000000..69d2146b244747 --- /dev/null +++ b/datahub-ranger-plugin/src/test/java/com/datahub/authorizer/plugin/ranger/TestDataHubRangerAuthPlugin.java @@ -0,0 +1,18 @@ +package com.datahub.authorizer.plugin.ranger; + +import org.junit.Test; + + +public class TestDataHubRangerAuthPlugin { + @Test(expected = UnsupportedOperationException.class) + public void testValidateConfig() throws Exception { + DataHubRangerAuthPlugin datahubRangerAuthPlugin = new DataHubRangerAuthPlugin(); + datahubRangerAuthPlugin.validateConfig(); + } + + @Test(expected = UnsupportedOperationException.class) + public void testLookupResource() throws Exception { + DataHubRangerAuthPlugin datahubRangerAuthPlugin = new DataHubRangerAuthPlugin(); + datahubRangerAuthPlugin.lookupResource(null); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java index e7c730d6e6d249..98e8de357f2322 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCli.java @@ -3,6 +3,7 @@ import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager; import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; import com.linkedin.datahub.upgrade.nocodecleanup.NoCodeCleanupUpgrade; +import com.linkedin.datahub.upgrade.removeunknownaspects.RemoveUnknownAspects; import com.linkedin.datahub.upgrade.restorebackup.RestoreBackup; import com.linkedin.datahub.upgrade.restoreindices.RestoreIndices; import java.util.List; @@ -44,12 +45,17 @@ private static final class Args { @Named("restoreBackup") private RestoreBackup restoreBackup; + @Inject + @Named("removeUnknownAspects") + private RemoveUnknownAspects removeUnknownAspects; + @Override public void run(String... cmdLineArgs) { _upgradeManager.register(noCodeUpgrade); _upgradeManager.register(noCodeCleanup); _upgradeManager.register(restoreIndices); _upgradeManager.register(restoreBackup); + _upgradeManager.register(removeUnknownAspects); final Args args = new Args(); new CommandLine(args).setCaseInsensitiveEnumValuesAllowed(true).parseArgs(cmdLineArgs); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeUtils.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeUtils.java index 0bfb9e2c50f99c..a6f3ef55604424 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeUtils.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeUtils.java @@ -15,14 +15,10 @@ public static Map> parseArgs(final List args) { if (args == null) { return Collections.emptyMap(); } - final Map> parsedArgs = new HashMap<>(); + for (final String arg : args) { - List parsedArg = Arrays.asList(arg.split(KEY_VALUE_DELIMITER)); - if (parsedArg.size() > 2) { - throw new RuntimeException("Failed to parse arguments provided as input to upgrade. Multiple '=' delimiters found in " - + String.format("%s", arg)); - } + List parsedArg = Arrays.asList(arg.split(KEY_VALUE_DELIMITER, 2)); parsedArgs.put(parsedArg.get(0), parsedArg.size() > 1 ? Optional.of(parsedArg.get(1)) : Optional.empty()); } return parsedArgs; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java new file mode 100644 index 00000000000000..cdc739efc416dd --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java @@ -0,0 +1,15 @@ +package com.linkedin.datahub.upgrade.config; + +import com.linkedin.datahub.upgrade.removeunknownaspects.RemoveUnknownAspects; +import com.linkedin.metadata.entity.EntityService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class RemoveUnknownAspectsConfig { + @Bean(name = "removeUnknownAspects") + public RemoveUnknownAspects removeUnknownAspects(EntityService entityService) { + return new RemoveUnknownAspects(entityService); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java new file mode 100644 index 00000000000000..b55d439745e691 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java @@ -0,0 +1,41 @@ +package com.linkedin.datahub.upgrade.removeunknownaspects; + +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.gms.factory.telemetry.TelemetryUtils; +import com.linkedin.metadata.entity.EntityService; +import java.util.HashMap; +import java.util.function.Function; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +@RequiredArgsConstructor +public class RemoveClientIdAspectStep implements UpgradeStep { + + private static final String INVALID_CLIENT_ID_ASPECT = "clientId"; + + private final EntityService _entityService; + + @Override + public String id() { + return this.getClass().getSimpleName(); + } + + @Override + public boolean skip(UpgradeContext context) { + return false; + } + + @Override + public Function executable() { + return upgradeContext -> { + _entityService.deleteAspect(TelemetryUtils.CLIENT_ID_URN, INVALID_CLIENT_ID_ASPECT, + new HashMap<>(), true); + return (UpgradeStepResult) new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + }; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java new file mode 100644 index 00000000000000..f8af69dba08653 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java @@ -0,0 +1,40 @@ +package com.linkedin.datahub.upgrade.removeunknownaspects; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.Upgrade; +import com.linkedin.datahub.upgrade.UpgradeCleanupStep; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.metadata.entity.EntityService; +import java.util.ArrayList; +import java.util.List; + + +public class RemoveUnknownAspects implements Upgrade { + + private final List _steps; + + public RemoveUnknownAspects(final EntityService entityService) { + _steps = buildSteps(entityService); + } + + @Override + public String id() { + return this.getClass().getSimpleName(); + } + + @Override + public List steps() { + return _steps; + } + + private List buildSteps(final EntityService entityService) { + final List steps = new ArrayList<>(); + steps.add(new RemoveClientIdAspectStep(entityService)); + return steps; + } + + @Override + public List cleanupSteps() { + return ImmutableList.of(); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java index 60c479fe3e6d98..7d4eebae677a0d 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java @@ -100,8 +100,8 @@ public Function executable() { AspectSpec aspectSpec = entitySpec.getAspectSpec(aspectName); if (aspectSpec == null) { context.report() - .addLine(String.format("Failed to find aspect with name %s associated with entity named %s", - aspectName, entityName)); + .addLine(String.format("Failed to find aspect with name %s associated with entity named %s", aspectName, + entityName)); continue; } @@ -135,11 +135,7 @@ public Function executable() { } } if (totalRowsMigrated != rowCount) { - context.report() - .addLine( - String.format("Number of MAEs sent %s does not equal the number of input rows %s...", totalRowsMigrated, - rowCount)); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); + context.report().addLine(String.format("Failed to send MAEs for %d rows...", rowCount - totalRowsMigrated)); } return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); }; diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle index 6e58e4aec13e98..3ffafa92442a71 100644 --- a/datahub-web-react/build.gradle +++ b/datahub-web-react/build.gradle @@ -74,7 +74,7 @@ clean { delete 'build' delete 'just' delete 'src/types.generated.ts' - delete 'src/graphql/*.generated.ts' + delete fileTree(dir: 'src/graphql', include: '*.generated.ts') } configurations { diff --git a/datahub-web-react/craco.config.js b/datahub-web-react/craco.config.js index 591c465a24c78b..e6774d8008138d 100644 --- a/datahub-web-react/craco.config.js +++ b/datahub-web-react/craco.config.js @@ -21,6 +21,13 @@ module.exports = { new CopyWebpackPlugin({ patterns: [{ from: 'src/images', to: 'platforms' }], }), + // Copy monaco-editor files to the build directory + new CopyWebpackPlugin({ + patterns: [ + { from: "node_modules/monaco-editor/min/vs/", to: "monaco-editor/vs" }, + { from: "node_modules/monaco-editor/min-maps/vs/", to: "monaco-editor/min-maps/vs" }, + ], + }), ], }, }, diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index 4b5272f00d574c..8bfacce879f5d8 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -22,6 +22,7 @@ "@testing-library/user-event": "^12.6.0", "@tommoor/remove-markdown": "^0.3.2", "@types/diff": "^5.0.0", + "@types/dompurify": "^2.3.3", "@types/jest": "^26.0.19", "@types/js-cookie": "^2.2.6", "@types/node": "^12.19.9", @@ -53,7 +54,9 @@ "cypress": "7.3.0", "d3-scale": "^3.3.0", "d3-time-format": "^3.0.0", + "deepmerge": "^4.2.2", "diff": "^5.0.0", + "dompurify": "^2.3.8", "dotenv": "^8.2.0", "faker": "5.5.3", "find-webpack": "2.2.1", @@ -66,6 +69,7 @@ "lodash.debounce": "^4.0.8", "miragejs": "^0.1.41", "moment-timezone": "^0.5.34", + "monaco-editor": "^0.28.1", "monaco-yaml": "^3.2.1", "query-string": "^6.13.8", "rc-table": "^7.13.1", @@ -74,6 +78,7 @@ "react-dom": "^17.0.0", "react-html-parser": "^2.0.2", "react-icons": "4.3.1", + "react-js-cron": "^2.1.0", "react-papaparse": "^3.16.1", "react-router": "^5.2.0", "react-router-dom": "^5.1.6", @@ -96,7 +101,7 @@ "start:mock": "yarn run generate && BROWSER=none REACT_APP_MOCK=true craco start", "start:e2e": "REACT_APP_MOCK=cy BROWSER=none PORT=3010 craco start", "ec2-dev": "yarn run generate && CI=true;export CI;BROWSER=none craco start", - "build": "export NODE_OPTIONS=--max_old_space_size=4096 && yarn run generate && CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/ dist/ && rm -r build/", + "build": "yarn run generate && CI=false REACT_APP_MOCK=false craco build && rm -rf dist/ && cp -r build/ dist/ && rm -r build/", "test": "craco test", "cy:run:ci": "cypress run", "pretest:e2e:ci": "yarn generate", diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx index 9ff31080c2ec9d..1da44040cea8d9 100644 --- a/datahub-web-react/src/App.tsx +++ b/datahub-web-react/src/App.tsx @@ -49,7 +49,7 @@ const errorLink = onError(({ graphQLErrors, networkError }) => { if (graphQLErrors && graphQLErrors.length) { const firstError = graphQLErrors[0]; const { extensions } = firstError; - console.log(firstError); + console.error(firstError); const errorCode = extensions && (extensions.code as number); // Fallback in case the calling component does not handle. message.error(`${firstError.message} (code ${errorCode})`, 3); diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 3ac3058f543fc8..4faa5ef4888a4f 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -26,6 +26,7 @@ import { RecommendationRenderType, RelationshipDirection, Container, + PlatformPrivileges, } from './types.generated'; import { GetTagDocument } from './graphql/tag.generated'; import { GetMlModelDocument } from './graphql/mlModel.generated'; @@ -66,6 +67,7 @@ const user1 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:corpuser:1', }, ], }, @@ -87,6 +89,13 @@ const user2 = { editableInfo: { pictureLink: null, }, + editableProperties: { + displayName: 'Test', + title: 'test', + pictureLink: null, + teams: [], + skills: [], + }, globalTags: { tags: [ { @@ -101,6 +110,7 @@ const user2 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:corpuser:3', }, ], }, @@ -144,9 +154,11 @@ export const dataset1 = { customProperties: [ { key: 'TestProperty', + associatedUrn: 'urn:li:dataset:1', value: 'My property value.', }, { + associatedUrn: 'urn:li:dataset:1', key: 'AnotherTestProperty', value: 'My other property value.', }, @@ -165,12 +177,14 @@ export const dataset1 = { owner: { ...user1, }, + associatedUrn: 'urn:li:dataset:1', type: 'DATAOWNER', }, { owner: { ...user2, }, + associatedUrn: 'urn:li:dataset:1', type: 'DELEGATE', }, ], @@ -207,10 +221,11 @@ export const dataset1 = { container: null, upstream: null, downstream: null, - health: null, + health: [], assertions: null, deprecation: null, testResults: null, + statsSummary: null, }; export const dataset2 = { @@ -252,6 +267,7 @@ export const dataset2 = { owner: { ...user1, }, + associatedUrn: 'urn:li:dataset:2', type: 'DATAOWNER', }, { @@ -259,6 +275,7 @@ export const dataset2 = { ...user2, }, type: 'DELEGATE', + associatedUrn: 'urn:li:dataset:2', }, ], lastModified: { @@ -288,11 +305,12 @@ export const dataset2 = { container: null, upstream: null, downstream: null, - health: null, + health: [], assertions: null, status: null, deprecation: null, testResults: null, + statsSummary: null, }; export const dataset3 = { @@ -319,7 +337,7 @@ export const dataset3 = { name: 'Yet Another Dataset', description: 'This and here we have yet another Dataset (YAN). Are there more?', origin: 'PROD', - customProperties: [{ key: 'propertyAKey', value: 'propertyAValue' }], + customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:3' }], externalUrl: 'https://data.hub', }, parentContainers: { @@ -340,12 +358,14 @@ export const dataset3 = { ...user1, }, type: 'DATAOWNER', + associatedUrn: 'urn:li:dataset:3', }, { owner: { ...user2, }, type: 'DELEGATE', + associatedUrn: 'urn:li:dataset:3', }, ], lastModified: { @@ -367,6 +387,7 @@ export const dataset3 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:dataset:3', }, ], }, @@ -385,6 +406,7 @@ export const dataset3 = { termSource: 'sample term source', }, }, + associatedUrn: 'urn:li:dataset:3', }, ], }, @@ -454,7 +476,6 @@ export const dataset3 = { primaryKeys: [], foreignKeys: [], }, - previousSchemaMetadata: null, editableSchemaMetadata: null, deprecation: null, usageStats: null, @@ -498,12 +519,14 @@ export const dataset3 = { container: null, lineage: null, relationships: null, - health: null, + health: [], assertions: null, status: null, readRuns: null, writeRuns: null, testResults: null, + siblings: null, + statsSummary: null, } as Dataset; export const dataset4 = { @@ -527,7 +550,7 @@ export const dataset5 = { name: 'Fifth Test Dataset', description: 'This and here we have yet another Dataset (YAN). Are there more?', origin: 'PROD', - customProperties: [{ key: 'propertyAKey', value: 'propertyAValue' }], + customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:5' }], externalUrl: 'https://data.hub', }, }; @@ -541,7 +564,7 @@ export const dataset6 = { qualifiedName: 'Fully Qualified Name of Sixth Test Dataset', description: 'This and here we have yet another Dataset (YAN). Are there more?', origin: 'PROD', - customProperties: [{ key: 'propertyAKey', value: 'propertyAValue' }], + customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:6' }], externalUrl: 'https://data.hub', }, }; @@ -801,12 +824,14 @@ const glossaryTerm1 = { owner: { ...user1, }, + associatedUrn: 'urn:li:glossaryTerm:1', type: 'DATAOWNER', }, { owner: { ...user2, }, + associatedUrn: 'urn:li:glossaryTerm:1', type: 'DELEGATE', }, ], @@ -851,7 +876,7 @@ const glossaryTerm2 = { { key: 'keyProperty', value: 'valueProperty', - __typename: 'StringMapEntry', + __typename: 'CustomPropertiesEntry', }, ], __typename: 'GlossaryTermInfo', @@ -868,7 +893,7 @@ const glossaryTerm2 = { { key: 'keyProperty', value: 'valueProperty', - __typename: 'StringMapEntry', + __typename: 'CustomPropertiesEntry', }, ], __typename: 'GlossaryTermProperties', @@ -923,7 +948,8 @@ const glossaryTerm3 = { { key: 'keyProperty', value: 'valueProperty', - __typename: 'StringMapEntry', + associatedUrn: 'urn:li:glossaryTerm:example.glossaryterm2', + __typename: 'CustomPropertiesEntry', }, ], __typename: 'GlossaryTermInfo', @@ -940,7 +966,8 @@ const glossaryTerm3 = { { key: 'keyProperty', value: 'valueProperty', - __typename: 'StringMapEntry', + associatedUrn: 'urn:li:glossaryTerm:example.glossaryterm2', + __typename: 'CustomPropertiesEntry', }, ], __typename: 'GlossaryTermProperties', @@ -1086,12 +1113,14 @@ export const dataFlow1 = { ...user1, }, type: 'DATAOWNER', + associatedUrn: 'urn:li:dataFlow:1', }, { owner: { ...user2, }, type: 'DELEGATE', + associatedUrn: 'urn:li:dataFlow:1', }, ], lastModified: { @@ -1112,11 +1141,20 @@ export const dataFlow1 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:dataFlow:1', }, ], }, platform: { - ...dataPlatform, + urn: 'urn:li:dataPlatform:airflow', + name: 'Airflow', + type: EntityType.DataPlatform, + properties: { + displayName: 'Airflow', + type: PlatformType.FileSystem, + datasetNameDelimiter: '.', + logoUrl: '', + }, }, domain: null, deprecation: null, @@ -1135,12 +1173,14 @@ export const dataJob1 = { owner: { ...user1, }, + associatedUrn: 'urn:li:dataJob:1', type: 'DATAOWNER', }, { owner: { ...user2, }, + associatedUrn: 'urn:li:dataJob:1', type: 'DELEGATE', }, ], @@ -1175,6 +1215,7 @@ export const dataJob1 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:dataJob:1', }, ], }, @@ -1212,12 +1253,14 @@ export const dataJob2 = { owner: { ...user1, }, + associatedUrn: 'urn:li:dataJob:2', type: 'DATAOWNER', }, { owner: { ...user2, }, + associatedUrn: 'urn:li:dataJob:2', type: 'DELEGATE', }, ], @@ -1252,6 +1295,7 @@ export const dataJob2 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:dataJob:2', }, ], }, @@ -1274,12 +1318,14 @@ export const dataJob3 = { owner: { ...user1, }, + associatedUrn: 'urn:li:dataJob:3', type: 'DATAOWNER', }, { owner: { ...user2, }, + associatedUrn: 'urn:li:dataJob:3', type: 'DELEGATE', }, ], @@ -1314,6 +1360,7 @@ export const dataJob3 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:dataJob:3', }, ], }, @@ -1361,12 +1408,14 @@ export const mlModel = { ...user1, }, type: 'DATAOWNER', + associatedUrn: 'urn:li:mlModel:(urn:li:dataPlatform:sagemaker,trustmodel,PROD)', }, { owner: { ...user2, }, type: 'DELEGATE', + associatedUrn: 'urn:li:mlModel:(urn:li:dataPlatform:sagemaker,trustmodel,PROD)', }, ], lastModified: { @@ -1389,6 +1438,7 @@ export const mlModel = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:mlModel:(urn:li:dataPlatform:sagemaker,trustmodel,PROD)', }, ], }, @@ -1446,12 +1496,14 @@ export const mlModelGroup = { owner: { ...user1, }, + associatedUrn: 'urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,another-group,PROD)', type: 'DATAOWNER', }, { owner: { ...user2, }, + associatedUrn: 'urn:li:mlModelGroup:(urn:li:dataPlatform:sagemaker,another-group,PROD)', type: 'DELEGATE', }, ], @@ -1752,7 +1804,7 @@ export const mocks = [ variables: { input: { query: 't', - limit: 30, + limit: 10, }, }, }, @@ -3082,6 +3134,17 @@ export const mocks = [ viewAnalytics: true, managePolicies: true, manageIdentities: true, + manageDomains: true, + manageTags: true, + createDomains: true, + createTags: true, + manageUserCredentials: true, + manageGlossaries: true, + manageTests: true, + manageTokens: true, + manageSecrets: true, + manageIngestion: true, + generatePersonalAccessTokens: true, }, }, }, @@ -3222,7 +3285,7 @@ export const mocks = [ types: [], query: '*', start: 0, - count: 20, + count: 6, filters: [], }, }, @@ -3288,3 +3351,20 @@ export const mocks = [ }, }, ]; + +export const platformPrivileges: PlatformPrivileges = { + viewAnalytics: true, + managePolicies: true, + manageIdentities: true, + generatePersonalAccessTokens: true, + manageDomains: true, + manageIngestion: true, + manageSecrets: true, + manageTokens: true, + manageTests: true, + manageGlossaries: true, + manageUserCredentials: true, + manageTags: true, + createTags: true, + createDomains: true, +}; diff --git a/datahub-web-react/src/MocksCustom.tsx b/datahub-web-react/src/MocksCustom.tsx index 189ea56f1bdd67..5466049948a9c0 100644 --- a/datahub-web-react/src/MocksCustom.tsx +++ b/datahub-web-react/src/MocksCustom.tsx @@ -98,7 +98,7 @@ export const dataset3 = { name: 'Yet Another Dataset', description: 'This and here we have yet another Dataset (YAN). Are there more?', origin: 'PROD', - customProperties: [{ key: 'propertyAKey', value: 'propertyAValue' }], + customProperties: [{ key: 'propertyAKey', value: 'propertyAValue', associatedUrn: 'urn:li:dataset:3' }], externalUrl: 'https://data.hub', }, parentContainers: { @@ -118,14 +118,44 @@ export const dataset3 = { { owner: { __typename: 'CorpUser', - ...user2, + urn: 'urn:li:corpuser:2', + type: 'CORP_USER', + username: 'jdoe', + info: { + __typename: 'CorpUserInfo', + active: true, + displayName: 'John Doe', + title: 'Software Engineer', + email: 'jdoe@linkedin.com', + firstName: null, + lastName: null, + fullName: 'John Doe', + }, + properties: { + __typename: 'CorpUserProperties', + active: true, + displayName: 'John Doe', + title: 'Software Engineer', + email: 'jdoe@linkedin.com', + firstName: null, + lastName: null, + fullName: 'John Doe', + }, + editableProperties: { + __typename: 'CorpUserEditableProperties', + displayName: 'sdas', + title: 'Software Engineer', + pictureLink: 'something', + email: 'sdas@domain.com', + }, }, - type: 'TECHNICAL_OWNER', - source: null, + type: 'DATAOWNER', + associatedUrn: 'urn:li:dataset:3', }, ], lastModified: { - time: 0, + __typename: 'AuditStamp', + time: 1581407189000, }, }, globalTags: { @@ -143,6 +173,7 @@ export const dataset3 = { colorHex: 'sample tag color', }, }, + associatedUrn: 'urn:li:dataset:3', }, ], }, @@ -161,6 +192,7 @@ export const dataset3 = { termSource: 'sample term source', }, }, + associatedUrn: 'urn:li:dataset:3', }, ], }, @@ -230,7 +262,6 @@ export const dataset3 = { primaryKeys: [], foreignKeys: [], }, - previousSchemaMetadata: null, editableSchemaMetadata: null, deprecation: null, usageStats: null, @@ -276,12 +307,14 @@ export const dataset3 = { }, lineage: null, relationships: null, - health: null, + health: [], assertions: null, status: null, readRuns: null, writeRuns: null, testResults: null, + siblings: null, + statsSummary: null, } as Dataset; /* diff --git a/datahub-web-react/src/app/ProtectedRoutes.tsx b/datahub-web-react/src/app/ProtectedRoutes.tsx index e9b14a0acb12e8..c7230c256caf10 100644 --- a/datahub-web-react/src/app/ProtectedRoutes.tsx +++ b/datahub-web-react/src/app/ProtectedRoutes.tsx @@ -1,46 +1,21 @@ import React from 'react'; -import { Switch, Route, Redirect } from 'react-router-dom'; +import { Switch, Route } from 'react-router-dom'; import { Layout } from 'antd'; -import { BrowseResultsPage } from './browse/BrowseResultsPage'; -import { EntityPage } from './entity/EntityPage'; -import { PageRoutes } from '../conf/Global'; -import { useEntityRegistry } from './useEntityRegistry'; import { HomePage } from './home/HomePage'; -import { SearchPage } from './search/SearchPage'; -import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage'; import AppConfigProvider from '../AppConfigProvider'; -import { ManageIngestionPage } from './ingest/ManageIngestionPage'; -import { ManageDomainsPage } from './domain/ManageDomainsPage'; -import BusinessGlossaryPage from './glossary/BusinessGlossaryPage'; -import { SettingsPage } from './settings/SettingsPage'; +import { SearchRoutes } from './SearchRoutes'; /** * Container for all views behind an authentication wall. */ export const ProtectedRoutes = (): JSX.Element => { - const entityRegistry = useEntityRegistry(); return ( } /> - {entityRegistry.getEntities().map((entity) => ( - } - /> - ))} - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> + } /> diff --git a/datahub-web-react/src/app/Routes.tsx b/datahub-web-react/src/app/Routes.tsx index ab94955e760a20..335298e603f174 100644 --- a/datahub-web-react/src/app/Routes.tsx +++ b/datahub-web-react/src/app/Routes.tsx @@ -2,6 +2,8 @@ import React from 'react'; import { Switch, Route, RouteProps, Redirect } from 'react-router-dom'; import { useReactiveVar } from '@apollo/client'; import { LogIn } from './auth/LogIn'; +import { SignUp } from './auth/SignUp'; +import { ResetCredentials } from './auth/ResetCredentials'; import { NoPageFound } from './shared/NoPageFound'; import { PageRoutes } from '../conf/Global'; import { isLoggedInVar } from './auth/checkAuthStatus'; @@ -33,11 +35,13 @@ export const Routes = (): JSX.Element => { return ( + + } /> } /> {/* Starting the react app locally opens /assets by default. For a smoother dev experience, we'll redirect to the homepage */} } exact /> - + ); }; diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx new file mode 100644 index 00000000000000..330060f5ac3152 --- /dev/null +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -0,0 +1,44 @@ +import React from 'react'; +import { Switch, Route, Redirect } from 'react-router-dom'; +import { NoPageFound } from './shared/NoPageFound'; +import { PageRoutes } from '../conf/Global'; +import { SearchablePage } from './search/SearchablePage'; +import { useEntityRegistry } from './useEntityRegistry'; +import { EntityPage } from './entity/EntityPage'; +import { BrowseResultsPage } from './browse/BrowseResultsPage'; +import { SearchPage } from './search/SearchPage'; +import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage'; +import { ManageDomainsPage } from './domain/ManageDomainsPage'; +import { ManageIngestionPage } from './ingest/ManageIngestionPage'; +import BusinessGlossaryPage from './glossary/BusinessGlossaryPage'; +import { SettingsPage } from './settings/SettingsPage'; + +/** + * Container for all searchable page routes + */ +export const SearchRoutes = (): JSX.Element => { + const entityRegistry = useEntityRegistry(); + return ( + + + {entityRegistry.getEntities().map((entity) => ( + } + /> + ))} + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + + ); +}; diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index e4ca1cf86bcabc..ff6f76d0f4459f 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -15,11 +15,14 @@ export enum EventType { EntityViewEvent, EntitySectionViewEvent, EntityActionEvent, + BatchEntityActionEvent, RecommendationImpressionEvent, RecommendationClickEvent, SearchAcrossLineageEvent, SearchAcrossLineageResultsViewEvent, DownloadAsCsvEvent, + SignUpEvent, + ResetCredentialsEvent, } /** @@ -40,6 +43,14 @@ export interface PageViewEvent extends BaseEvent { type: EventType.PageViewEvent; } +/** + * Logged on successful new user sign up. + */ +export interface SignUpEvent extends BaseEvent { + type: EventType.SignUpEvent; + title: string; +} + /** * Logged on user successful login. */ @@ -54,6 +65,13 @@ export interface LogOutEvent extends BaseEvent { type: EventType.LogOutEvent; } +/** + * Logged on user resetting their credentials + */ +export interface ResetCredentialsEvent extends BaseEvent { + type: EventType.ResetCredentialsEvent; +} + /** * Logged on user successful search query. */ @@ -139,10 +157,16 @@ export const EntityActionType = { export interface EntityActionEvent extends BaseEvent { type: EventType.EntityActionEvent; actionType: string; - entityType: EntityType; + entityType?: EntityType; entityUrn: string; } +export interface BatchEntityActionEvent extends BaseEvent { + type: EventType.BatchEntityActionEvent; + actionType: string; + entityUrns: string[]; +} + export interface RecommendationImpressionEvent extends BaseEvent { type: EventType.RecommendationImpressionEvent; renderId: string; // TODO : Determine whether we need a render id to join with click event. @@ -189,8 +213,10 @@ export interface DownloadAsCsvEvent extends BaseEvent { */ export type Event = | PageViewEvent + | SignUpEvent | LogInEvent | LogOutEvent + | ResetCredentialsEvent | SearchEvent | SearchResultsViewEvent | SearchResultClickEvent @@ -202,4 +228,5 @@ export type Event = | SearchAcrossLineageEvent | SearchAcrossLineageResultsViewEvent | DownloadAsCsvEvent - | RecommendationClickEvent; + | RecommendationClickEvent + | BatchEntityActionEvent; diff --git a/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx b/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx index 7f019fb8322fca..76bd2e34eddeaa 100644 --- a/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx +++ b/datahub-web-react/src/app/analyticsDashboard/components/AnalyticsPage.tsx @@ -2,8 +2,6 @@ import React, { useState } from 'react'; import styled from 'styled-components'; import { Alert, Divider, Input, Select } from 'antd'; import { SearchOutlined } from '@ant-design/icons'; - -import { SearchablePage } from '../../search/SearchablePage'; import { ChartGroup } from './ChartGroup'; import { useGetAnalyticsChartsQuery, useGetMetadataAnalyticsChartsQuery } from '../../../graphql/analytics.generated'; import { useGetHighlightsQuery } from '../../../graphql/highlights.generated'; @@ -85,7 +83,7 @@ export const AnalyticsPage = () => { }); return ( - + <> {highlightLoading && ( @@ -179,6 +177,6 @@ export const AnalyticsPage = () => { ))} - + ); }; diff --git a/datahub-web-react/src/app/auth/ResetCredentials.tsx b/datahub-web-react/src/app/auth/ResetCredentials.tsx new file mode 100644 index 00000000000000..310a0746964ed3 --- /dev/null +++ b/datahub-web-react/src/app/auth/ResetCredentials.tsx @@ -0,0 +1,169 @@ +import React, { useCallback, useState } from 'react'; +import { Input, Button, Form, message, Image } from 'antd'; +import { UserOutlined, LockOutlined } from '@ant-design/icons'; +import { useReactiveVar } from '@apollo/client'; +import styled, { useTheme } from 'styled-components'; +import { Redirect } from 'react-router'; +import styles from './login.module.css'; +import { Message } from '../shared/Message'; +import { isLoggedInVar } from './checkAuthStatus'; +import analytics, { EventType } from '../analytics'; +import { useAppConfig } from '../useAppConfig'; +import { PageRoutes } from '../../conf/Global'; +import useGetResetTokenFromUrlParams from './useGetResetTokenFromUrlParams'; + +type FormValues = { + email: string; + password: string; + confirmPassword: string; +}; + +const FormInput = styled(Input)` + &&& { + height: 32px; + font-size: 12px; + border: 1px solid #555555; + border-radius: 5px; + background-color: transparent; + color: white; + line-height: 1.5715; + } + > .ant-input { + color: white; + font-size: 14px; + background-color: transparent; + } + > .ant-input:hover { + color: white; + font-size: 14px; + background-color: transparent; + } +`; + +export type ResetCredentialsProps = Record; + +export const ResetCredentials: React.VFC = () => { + const isLoggedIn = useReactiveVar(isLoggedInVar); + const resetToken = useGetResetTokenFromUrlParams(); + + const themeConfig = useTheme(); + const [loading, setLoading] = useState(false); + + const { refreshContext } = useAppConfig(); + + const handleResetCredentials = useCallback( + (values: FormValues) => { + setLoading(true); + const requestOptions = { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + email: values.email, + password: values.password, + resetToken, + }), + }; + fetch('/resetNativeUserCredentials', requestOptions) + .then(async (response) => { + if (!response.ok) { + const data = await response.json(); + const error = (data && data.message) || response.status; + return Promise.reject(error); + } + isLoggedInVar(true); + refreshContext(); + analytics.event({ type: EventType.ResetCredentialsEvent }); + return Promise.resolve(); + }) + .catch((error) => { + message.error(`Failed to log in! ${error}`); + }) + .finally(() => setLoading(false)); + }, + [refreshContext, resetToken], + ); + + if (isLoggedIn && !loading) { + return ; + } + + return ( +

+
+
+ +
+
+ {loading && } +
+ Email} + > + } data-testid="email" /> + + ({ + validator() { + if (getFieldValue('password').length < 8) { + return Promise.reject( + new Error('Your password is less than 8 characters!'), + ); + } + return Promise.resolve(); + }, + }), + ]} + name="password" + // eslint-disable-next-line jsx-a11y/label-has-associated-control + label={} + > + } type="password" data-testid="password" /> + + ({ + validator() { + if (getFieldValue('confirmPassword') !== getFieldValue('password')) { + return Promise.reject(new Error('Your passwords do not match!')); + } + return Promise.resolve(); + }, + }), + ]} + name="confirmPassword" + // eslint-disable-next-line jsx-a11y/label-has-associated-control + label={} + > + } type="password" data-testid="confirmPassword" /> + + + {({ getFieldsValue }) => { + const { email, password, confirmPassword } = getFieldsValue(); + const fieldsAreNotEmpty = !!email && !!password && !!confirmPassword; + const passwordsMatch = password === confirmPassword; + const formIsComplete = fieldsAreNotEmpty && passwordsMatch; + return ( + + ); + }} + +
+
+
+
+ ); +}; diff --git a/datahub-web-react/src/app/auth/SignUp.tsx b/datahub-web-react/src/app/auth/SignUp.tsx new file mode 100644 index 00000000000000..ffda9ce85c63c9 --- /dev/null +++ b/datahub-web-react/src/app/auth/SignUp.tsx @@ -0,0 +1,209 @@ +import React, { useCallback, useState } from 'react'; +import { Input, Button, Form, message, Image, Select } from 'antd'; +import { UserOutlined, LockOutlined } from '@ant-design/icons'; +import { useReactiveVar } from '@apollo/client'; +import styled, { useTheme } from 'styled-components/macro'; +import { Redirect } from 'react-router'; +import styles from './login.module.css'; +import { Message } from '../shared/Message'; +import { isLoggedInVar } from './checkAuthStatus'; +import analytics, { EventType } from '../analytics'; +import { useAppConfig } from '../useAppConfig'; +import { PageRoutes } from '../../conf/Global'; +import useGetInviteTokenFromUrlParams from './useGetInviteTokenFromUrlParams'; + +type FormValues = { + fullName: string; + email: string; + password: string; + confirmPassword: string; + title: string; +}; + +const FormInput = styled(Input)` + &&& { + height: 32px; + font-size: 12px; + border: 1px solid #555555; + border-radius: 5px; + background-color: transparent; + color: white; + line-height: 1.5715; + } + > .ant-input { + color: white; + font-size: 14px; + background-color: transparent; + } + > .ant-input:hover { + color: white; + font-size: 14px; + background-color: transparent; + } +`; + +const TitleSelector = styled(Select)` + .ant-select-selector { + color: white; + border: 1px solid #555555 !important; + background-color: transparent !important; + } + .ant-select-arrow { + color: white; + } +`; + +export type SignUpProps = Record; + +export const SignUp: React.VFC = () => { + const isLoggedIn = useReactiveVar(isLoggedInVar); + const inviteToken = useGetInviteTokenFromUrlParams(); + + const themeConfig = useTheme(); + const [loading, setLoading] = useState(false); + + const { refreshContext } = useAppConfig(); + + const handleSignUp = useCallback( + (values: FormValues) => { + setLoading(true); + const requestOptions = { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + fullName: values.fullName, + email: values.email, + password: values.password, + title: values.title, + inviteToken, + }), + }; + fetch('/signUp', requestOptions) + .then(async (response) => { + if (!response.ok) { + const data = await response.json(); + const error = (data && data.message) || response.status; + return Promise.reject(error); + } + isLoggedInVar(true); + refreshContext(); + analytics.event({ type: EventType.SignUpEvent, title: values.title }); + return Promise.resolve(); + }) + .catch((error) => { + message.error(`Failed to log in! ${error}`); + }) + .finally(() => setLoading(false)); + }, + [refreshContext, inviteToken], + ); + + if (isLoggedIn && !loading) { + return ; + } + + return ( +
+
+
+ +
+
+ {loading && } +
+ Email} + > + } data-testid="email" /> + + Full Name} + > + } data-testid="name" /> + + ({ + validator() { + if (getFieldValue('password').length < 8) { + return Promise.reject( + new Error('Your password is less than 8 characters!'), + ); + } + return Promise.resolve(); + }, + }), + ]} + name="password" + // eslint-disable-next-line jsx-a11y/label-has-associated-control + label={} + > + } type="password" data-testid="password" /> + + ({ + validator() { + if (getFieldValue('confirmPassword') !== getFieldValue('password')) { + return Promise.reject(new Error('Your passwords do not match!')); + } + return Promise.resolve(); + }, + }), + ]} + name="confirmPassword" + // eslint-disable-next-line jsx-a11y/label-has-associated-control + label={} + > + } type="password" data-testid="confirmPassword" /> + + Title} + > + + Data Analyst + Data Engineer + Data Scientist + Software Engineer + Manager + Product Manager + Other + + + + {({ getFieldsValue }) => { + const { fullName, email, password, confirmPassword, title } = getFieldsValue(); + const fieldsAreNotEmpty = + !!fullName && !!email && !!password && !!confirmPassword && !!title; + const passwordsMatch = password === confirmPassword; + const formIsComplete = fieldsAreNotEmpty && passwordsMatch; + return ( + + ); + }} + +
+
+
+
+ ); +}; diff --git a/datahub-web-react/src/app/auth/useGetInviteTokenFromUrlParams.tsx b/datahub-web-react/src/app/auth/useGetInviteTokenFromUrlParams.tsx new file mode 100644 index 00000000000000..7654b7c8b4ef67 --- /dev/null +++ b/datahub-web-react/src/app/auth/useGetInviteTokenFromUrlParams.tsx @@ -0,0 +1,9 @@ +import * as QueryString from 'query-string'; +import { useLocation } from 'react-router-dom'; + +export default function useGetInviteTokenFromUrlParams() { + const location = useLocation(); + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const inviteToken: string = params.invite_token as string; + return inviteToken; +} diff --git a/datahub-web-react/src/app/auth/useGetResetTokenFromUrlParams.tsx b/datahub-web-react/src/app/auth/useGetResetTokenFromUrlParams.tsx new file mode 100644 index 00000000000000..ef9c3409f74af7 --- /dev/null +++ b/datahub-web-react/src/app/auth/useGetResetTokenFromUrlParams.tsx @@ -0,0 +1,9 @@ +import * as QueryString from 'query-string'; +import { useLocation } from 'react-router-dom'; + +export default function useGetResetTokenFromUrlParams() { + const location = useLocation(); + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const resetToken: string = params.reset_token as string; + return resetToken; +} diff --git a/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx b/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx index 5312f4e09d3ab2..2a5d9d9d251e3d 100644 --- a/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx +++ b/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx @@ -1,11 +1,10 @@ import { Result } from 'antd'; import React from 'react'; -import { SearchablePage } from '../search/SearchablePage'; export const UnauthorizedPage = () => { return ( - + <> - + ); }; diff --git a/datahub-web-react/src/app/browse/BrowsableEntityPage.tsx b/datahub-web-react/src/app/browse/BrowsableEntityPage.tsx index a16af634611c56..4cb9845d008b39 100644 --- a/datahub-web-react/src/app/browse/BrowsableEntityPage.tsx +++ b/datahub-web-react/src/app/browse/BrowsableEntityPage.tsx @@ -1,6 +1,5 @@ import * as React from 'react'; import { Affix } from 'antd'; -import { SearchablePage } from '../search/SearchablePage'; import { LegacyBrowsePath } from './LegacyBrowsePath'; import { useGetBrowsePathsQuery } from '../../graphql/browse.generated'; import { EntityType } from '../../types.generated'; @@ -26,7 +25,7 @@ export const BrowsableEntityPage = ({ const { data } = useGetBrowsePathsQuery({ variables: { input: { urn: _urn, type: _type } } }); return ( - + <> {data && data.browsePaths && data.browsePaths.length > 0 && ( )} {_children} - + ); }; diff --git a/datahub-web-react/src/app/browse/BrowseResultsPage.tsx b/datahub-web-react/src/app/browse/BrowseResultsPage.tsx index 485667b8f26342..aab4d3505d82d0 100644 --- a/datahub-web-react/src/app/browse/BrowseResultsPage.tsx +++ b/datahub-web-react/src/app/browse/BrowseResultsPage.tsx @@ -4,7 +4,6 @@ import * as QueryString from 'query-string'; import { Affix, Alert } from 'antd'; import { BrowseCfg } from '../../conf'; import { BrowseResults } from './BrowseResults'; -import { SearchablePage } from '../search/SearchablePage'; import { useGetBrowseResultsQuery } from '../../graphql/browse.generated'; import { LegacyBrowsePath } from './LegacyBrowsePath'; import { PageRoutes } from '../../conf/Global'; @@ -56,7 +55,7 @@ export const BrowseResultsPage = () => { } return ( - + <> @@ -74,6 +73,6 @@ export const BrowseResultsPage = () => { onChangePage={onChangePage} /> )} - + ); }; diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index 984c226d365079..3306104c1698d8 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -16,14 +16,13 @@ const ClickableTag = styled(Tag)` `; type Props = { - visible: boolean; onClose: () => void; onCreate: (id: string | undefined, name: string, description: string) => void; }; const SUGGESTED_DOMAIN_NAMES = ['Engineering', 'Marketing', 'Sales', 'Product']; -export default function CreateDomainModal({ visible, onClose, onCreate }: Props) { +export default function CreateDomainModal({ onClose, onCreate }: Props) { const [stagedName, setStagedName] = useState(''); const [stagedDescription, setStagedDescription] = useState(''); const [stagedId, setStagedId] = useState(undefined); @@ -66,7 +65,7 @@ export default function CreateDomainModal({ visible, onClose, onCreate }: Props) return ( diff --git a/datahub-web-react/src/app/domain/DomainListItem.tsx b/datahub-web-react/src/app/domain/DomainListItem.tsx index c9f82375bb0d09..1dc56aae2dff34 100644 --- a/datahub-web-react/src/app/domain/DomainListItem.tsx +++ b/datahub-web-react/src/app/domain/DomainListItem.tsx @@ -6,6 +6,8 @@ import { IconStyleType } from '../entity/Entity'; import { Domain, EntityType } from '../../types.generated'; import { useEntityRegistry } from '../useEntityRegistry'; import AvatarsGroup from '../shared/avatar/AvatarsGroup'; +import EntityDropdown from '../entity/shared/EntityDropdown'; +import { EntityMenuItems } from '../entity/shared/EntityDropdown/EntityDropdown'; const DomainItemContainer = styled.div` display: flex; @@ -28,9 +30,10 @@ const DomainNameContainer = styled.div` type Props = { domain: Domain; + onDelete?: () => void; }; -export default function DomainListItem({ domain }: Props) { +export default function DomainListItem({ domain, onDelete }: Props) { const entityRegistry = useEntityRegistry(); const displayName = entityRegistry.getDisplayName(EntityType.Domain, domain); const logoIcon = entityRegistry.getIcon(EntityType.Domain, 12, IconStyleType.ACCENT); @@ -54,6 +57,14 @@ export default function DomainListItem({ domain }: Props) { {owners && owners.length > 0 && ( )} + ); diff --git a/datahub-web-react/src/app/domain/DomainsList.tsx b/datahub-web-react/src/app/domain/DomainsList.tsx index 4c2d97dbeba691..e689dc5bc5ac57 100644 --- a/datahub-web-react/src/app/domain/DomainsList.tsx +++ b/datahub-web-react/src/app/domain/DomainsList.tsx @@ -50,6 +50,7 @@ export const DomainsList = () => { const [page, setPage] = useState(1); const [isCreatingDomain, setIsCreatingDomain] = useState(false); + const [removedUrns, setRemovedUrns] = useState([]); const pageSize = DEFAULT_PAGE_SIZE; const start = (page - 1) * pageSize; @@ -70,13 +71,20 @@ export const DomainsList = () => { const domains = (data?.listDomains?.domains || []).sort( (a, b) => (b.entities?.total || 0) - (a.entities?.total || 0), ); + const filteredDomains = domains.filter((domain) => !removedUrns.includes(domain.urn)); const onChangePage = (newPage: number) => { setPage(newPage); }; - // TODO: Handle robust deleting of domains. - + const handleDelete = (urn: string) => { + // Hack to deal with eventual consistency. + const newRemovedUrns = [...removedUrns, urn]; + setRemovedUrns(newRemovedUrns); + setTimeout(function () { + refetch?.(); + }, 3000); + }; return ( <> {!data && loading && } @@ -110,8 +118,10 @@ export const DomainsList = () => { locale={{ emptyText: , }} - dataSource={domains} - renderItem={(item: any) => } + dataSource={filteredDomains} + renderItem={(item: any) => ( + handleDelete(item.urn)} /> + )} /> @@ -130,16 +140,17 @@ export const DomainsList = () => { /> - setIsCreatingDomain(false)} - onCreate={() => { - // Hack to deal with eventual consistency. - setTimeout(function () { - refetch?.(); - }, 2000); - }} - /> + {isCreatingDomain && ( + setIsCreatingDomain(false)} + onCreate={() => { + // Hack to deal with eventual consistency. + setTimeout(function () { + refetch?.(); + }, 2000); + }} + /> + )} ); diff --git a/datahub-web-react/src/app/domain/ManageDomainsPage.tsx b/datahub-web-react/src/app/domain/ManageDomainsPage.tsx index e6800dc0695055..6172ac0246f58e 100644 --- a/datahub-web-react/src/app/domain/ManageDomainsPage.tsx +++ b/datahub-web-react/src/app/domain/ManageDomainsPage.tsx @@ -1,7 +1,6 @@ import { Typography } from 'antd'; import React from 'react'; import styled from 'styled-components'; -import { SearchablePage } from '../search/SearchablePage'; import { DomainsList } from './DomainsList'; const PageContainer = styled.div` @@ -24,18 +23,16 @@ const ListContainer = styled.div``; export const ManageDomainsPage = () => { return ( - - - - Domains - - View your DataHub Domains. Take administrative actions. - - - - - - - + + + Domains + + View your DataHub Domains. Take administrative actions. + + + + + + ); }; diff --git a/datahub-web-react/src/app/entity/Entity.tsx b/datahub-web-react/src/app/entity/Entity.tsx index c307605646fa15..61c4a27598940d 100644 --- a/datahub-web-react/src/app/entity/Entity.tsx +++ b/datahub-web-react/src/app/entity/Entity.tsx @@ -40,6 +40,36 @@ export enum IconStyleType { SVG, } +/** + * A standard set of Entity Capabilities that span across entity types. + */ +export enum EntityCapabilityType { + /** + * Ownership of an entity + */ + OWNERS, + /** + * Adding a glossary term to the entity + */ + GLOSSARY_TERMS, + /** + * Adding a tag to an entity + */ + TAGS, + /** + * Assigning the entity to a domain + */ + DOMAINS, + /** + * Deprecating an entity + */ + DEPRECATION, + /** + * Soft deleting an entity + */ + SOFT_DELETE, +} + /** * Base interface used for authoring DataHub Entities on the client side. * @@ -124,4 +154,9 @@ export interface Entity { * Returns generic entity properties for the entity */ getGenericEntityProperties: (data: T) => GenericEntityProperties | null; + + /** + * Returns the supported features for the entity + */ + supportedCapabilities: () => Set; } diff --git a/datahub-web-react/src/app/entity/EntityPage.tsx b/datahub-web-react/src/app/entity/EntityPage.tsx index 6a4015a21e2fa3..6ba12e8e6242d8 100644 --- a/datahub-web-react/src/app/entity/EntityPage.tsx +++ b/datahub-web-react/src/app/entity/EntityPage.tsx @@ -5,7 +5,6 @@ import { EntityType } from '../../types.generated'; import { BrowsableEntityPage } from '../browse/BrowsableEntityPage'; import LineageExplorer from '../lineage/LineageExplorer'; import useIsLineageMode from '../lineage/utils/useIsLineageMode'; -import { SearchablePage } from '../search/SearchablePage'; import { useEntityRegistry } from '../useEntityRegistry'; import analytics, { EventType } from '../analytics'; import { decodeUrn } from './shared/utils'; @@ -32,7 +31,6 @@ export const EntityPage = ({ entityType }: Props) => { const entity = entityRegistry.getEntity(entityType); const isBrowsable = entity.isBrowseEnabled(); const isLineageSupported = entity.isLineageEnabled(); - const ContainerPage = isBrowsable || isLineageSupported ? BrowsableEntityPage : SearchablePage; const isLineageMode = useIsLineageMode(); const authenticatedUserUrn = useGetAuthenticatedUserUrn(); const { loading, error, data } = useGetGrantedPrivilegesQuery({ @@ -74,8 +72,8 @@ export const EntityPage = ({ entityType }: Props) => { {error && } {data && !canViewEntityPage && } {canViewEntityPage && - ((showNewPage && {entityRegistry.renderProfile(entityType, urn)}) || ( - {entityRegistry.renderProfile(entityType, urn)}) || ( + { ) : ( entityRegistry.renderProfile(entityType, urn) )} - + ))} ); diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index 2714e1f75f974f..6aa24d9cf2196e 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -1,8 +1,8 @@ import { Entity as EntityInterface, EntityType, SearchResult } from '../../types.generated'; import { FetchedEntity } from '../lineage/types'; -import { Entity, IconStyleType, PreviewType } from './Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from './Entity'; import { GenericEntityProperties } from './shared/types'; -import { urlEncodeUrn } from './shared/utils'; +import { dictToQueryStringParams, urlEncodeUrn } from './shared/utils'; function validatedGet(key: K, map: Map): V { if (map.has(key)) { @@ -78,8 +78,8 @@ export default class EntityRegistry { return entity.getPathName(); } - getEntityUrl(type: EntityType, urn: string): string { - return `/${this.getPathName(type)}/${urlEncodeUrn(urn)}`; + getEntityUrl(type: EntityType, urn: string, params?: Record): string { + return `/${this.getPathName(type)}/${urlEncodeUrn(urn)}${params ? `?${dictToQueryStringParams(params)}` : ''}`; } getTypeFromPathName(pathName: string): EntityType { @@ -128,6 +128,7 @@ export default class EntityRegistry { entity: relationship.entity as EntityInterface, type: (relationship.entity as EntityInterface).type, })), + numDownstreamChildren: genericEntityProperties?.downstream?.total, upstreamChildren: genericEntityProperties?.upstream?.relationships ?.filter((relationship) => relationship.entity) // eslint-disable-next-line @typescript-eslint/dot-notation @@ -136,7 +137,9 @@ export default class EntityRegistry { entity: relationship.entity as EntityInterface, type: (relationship.entity as EntityInterface).type, })), + numUpstreamChildren: genericEntityProperties?.upstream?.total, status: genericEntityProperties?.status, + siblingPlatforms: genericEntityProperties?.siblingPlatforms, } as FetchedEntity) || undefined ); } @@ -150,4 +153,17 @@ export default class EntityRegistry { const entity = validatedGet(type, this.entityTypeToEntity); return entity.getGenericEntityProperties(data); } + + getSupportedEntityCapabilities(type: EntityType): Set { + const entity = validatedGet(type, this.entityTypeToEntity); + return entity.supportedCapabilities(); + } + + getTypesWithSupportedCapabilities(capability: EntityCapabilityType): Set { + return new Set( + this.getEntities() + .filter((entity) => entity.supportedCapabilities().has(capability)) + .map((entity) => entity.type), + ); + } } diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx index 142ee8d7231fb8..44cdd04e635f44 100644 --- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx +++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx @@ -1,7 +1,7 @@ import { LineChartOutlined } from '@ant-design/icons'; import * as React from 'react'; -import { Chart, EntityType, PlatformType, SearchResult } from '../../../types.generated'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Chart, EntityType, SearchResult } from '../../../types.generated'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { ChartPreview } from './preview/ChartPreview'; import { GetChartQuery, useGetChartQuery, useUpdateChartMutation } from '../../../graphql/chart.generated'; import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; @@ -14,9 +14,10 @@ import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; import { ChartInputsTab } from '../shared/tabs/Entity/ChartInputsTab'; import { ChartDashboardsTab } from '../shared/tabs/Entity/ChartDashboardsTab'; import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { capitalizeFirstLetter } from '../../shared/textUtil'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; +import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; +import { ChartStatsSummarySubHeader } from './profile/stats/ChartStatsSummarySubHeader'; /** * Definition of the DataHub Chart entity. @@ -71,6 +72,9 @@ export class ChartEntity implements Entity { useUpdateQuery={useUpdateChartMutation} getOverrideProperties={this.getOverridePropertiesFromEntity} headerDropdownItems={new Set([EntityMenuItems.COPY_URL, EntityMenuItems.UPDATE_DEPRECATION])} + subHeader={{ + component: ChartStatsSummarySubHeader, + }} tabs={[ { name: 'Documentation', @@ -80,6 +84,19 @@ export class ChartEntity implements Entity { name: 'Properties', component: PropertiesTab, }, + { + name: 'Lineage', + component: LineageTab, + display: { + visible: (_, _1) => true, + enabled: (_, chart: GetChartQuery) => { + return ( + (chart?.chart?.upstream?.total || 0) > 0 || (chart?.chart?.downstream?.total || 0) > 0 + ); + }, + }, + }, + { name: 'Inputs', component: ChartInputsTab, @@ -120,23 +137,11 @@ export class ChartEntity implements Entity { getOverridePropertiesFromEntity = (chart?: Chart | null): GenericEntityProperties => { // TODO: Get rid of this once we have correctly formed platform coming back. - const tool = chart?.tool || ''; const name = chart?.properties?.name; const externalUrl = chart?.properties?.externalUrl; return { name, externalUrl, - platform: { - urn: `urn:li:dataPlatform:(${tool})`, - type: EntityType.DataPlatform, - name: tool, - properties: { - logoUrl: chart?.platform?.properties?.logoUrl, - displayName: capitalizeFirstLetter(tool), - type: PlatformType.Others, - datasetNameDelimiter: '.', - }, - }, }; }; @@ -144,7 +149,7 @@ export class ChartEntity implements Entity { return ( { tags={data?.globalTags || undefined} glossaryTerms={data?.glossaryTerms} logoUrl={data?.platform?.properties?.logoUrl} - domain={data.domain} + domain={data.domain?.domain} parentContainers={data.parentContainers} /> ); @@ -163,7 +168,7 @@ export class ChartEntity implements Entity { return ( { glossaryTerms={data?.glossaryTerms} insights={result.insights} logoUrl={data?.platform?.properties?.logoUrl || ''} - domain={data.domain} + domain={data.domain?.domain} + deprecation={data.deprecation} + statsSummary={data.statsSummary} + lastUpdatedMs={data.properties?.lastModified?.time} + createdMs={data.properties?.created?.time} + externalUrl={data.properties?.externalUrl} /> ); }; @@ -184,7 +194,7 @@ export class ChartEntity implements Entity { name: entity.properties?.name || '', type: EntityType.Chart, icon: entity?.platform?.properties?.logoUrl || '', - platform: entity.tool, + platform: entity?.platform.properties?.displayName || entity?.platform.name, }; }; @@ -199,4 +209,15 @@ export class ChartEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx b/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx index d84f82e01ba0d7..6e7c3d9495c1ca 100644 --- a/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx +++ b/datahub-web-react/src/app/entity/chart/preview/ChartPreview.tsx @@ -9,10 +9,14 @@ import { Owner, SearchInsight, ParentContainersResult, + Deprecation, + ChartStatsSummary, } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { capitalizeFirstLetter } from '../../../shared/textUtil'; +import { IconStyleType } from '../../Entity'; +import { ChartStatsSummary as ChartStatsSummaryView } from '../shared/ChartStatsSummary'; export const ChartPreview = ({ urn, @@ -28,6 +32,11 @@ export const ChartPreview = ({ container, insights, logoUrl, + deprecation, + statsSummary, + lastUpdatedMs, + createdMs, + externalUrl, parentContainers, }: { urn: string; @@ -43,6 +52,11 @@ export const ChartPreview = ({ container?: Container | null; insights?: Array | null; logoUrl?: string | null; + deprecation?: Deprecation | null; + statsSummary?: ChartStatsSummary | null; + lastUpdatedMs?: number | null; + createdMs?: number | null; + externalUrl?: string | null; parentContainers?: ParentContainersResult | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); @@ -54,6 +68,7 @@ export const ChartPreview = ({ name={name || ''} description={description || ''} type="Chart" + typeIcon={entityRegistry.getIcon(EntityType.Chart, 14, IconStyleType.ACCENT)} logoUrl={logoUrl || ''} platform={capitalizedPlatform} platformInstanceId={platformInstanceId} @@ -65,6 +80,16 @@ export const ChartPreview = ({ container={container || undefined} insights={insights} parentContainers={parentContainers} + deprecation={deprecation} + externalUrl={externalUrl} + subHeader={ + + } /> ); }; diff --git a/datahub-web-react/src/app/entity/chart/profile/stats/ChartStatsSummarySubHeader.tsx b/datahub-web-react/src/app/entity/chart/profile/stats/ChartStatsSummarySubHeader.tsx new file mode 100644 index 00000000000000..8fbabe84297aec --- /dev/null +++ b/datahub-web-react/src/app/entity/chart/profile/stats/ChartStatsSummarySubHeader.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { ChartStatsSummary as ChartStatsSummaryObj } from '../../../../../types.generated'; +import { useBaseEntity } from '../../../shared/EntityContext'; +import { GetChartQuery } from '../../../../../graphql/chart.generated'; +import { ChartStatsSummary } from '../../shared/ChartStatsSummary'; + +export const ChartStatsSummarySubHeader = () => { + const result = useBaseEntity(); + const chart = result?.chart; + const maybeStatsSummary = chart?.statsSummary as ChartStatsSummaryObj; + const viewCount = maybeStatsSummary?.viewCount; + const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days; + const lastUpdatedMs = chart?.properties?.lastModified?.time; + const createdMs = chart?.properties?.created?.time; + + return ( + + ); +}; diff --git a/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx b/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx new file mode 100644 index 00000000000000..29b58ba756fc5e --- /dev/null +++ b/datahub-web-react/src/app/entity/chart/shared/ChartStatsSummary.tsx @@ -0,0 +1,79 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Popover, Tooltip } from 'antd'; +import { ClockCircleOutlined, EyeOutlined, TeamOutlined, QuestionCircleOutlined } from '@ant-design/icons'; +import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber'; +import { ANTD_GRAY } from '../../shared/constants'; +import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils'; +import { StatsSummary } from '../../shared/components/styled/StatsSummary'; + +const StatText = styled.span` + color: ${ANTD_GRAY[8]}; +`; + +const HelpIcon = styled(QuestionCircleOutlined)` + color: ${ANTD_GRAY[7]}; + padding-left: 4px; +`; + +type Props = { + chartCount?: number | null; + viewCount?: number | null; + uniqueUserCountLast30Days?: number | null; + lastUpdatedMs?: number | null; + createdMs?: number | null; +}; + +export const ChartStatsSummary = ({ + chartCount, + viewCount, + uniqueUserCountLast30Days, + lastUpdatedMs, + createdMs, +}: Props) => { + const statsViews = [ + (!!chartCount && ( + + {chartCount} charts + + )) || + undefined, + (!!viewCount && ( + + + {formatNumberWithoutAbbreviation(viewCount)} views + + )) || + undefined, + (!!uniqueUserCountLast30Days && ( + + + {formatNumberWithoutAbbreviation(uniqueUserCountLast30Days)} unique users + + )) || + undefined, + (!!lastUpdatedMs && ( + + {createdMs &&
Created on {toLocalDateTimeString(createdMs)}.
} +
+ Changed on {toLocalDateTimeString(lastUpdatedMs)}.{' '} + + + +
+ + } + > + + + Changed {toRelativeTimeString(lastUpdatedMs)} + + + )) || + undefined, + ].filter((stat) => stat !== undefined); + + return <>{statsViews.length > 0 && }; +}; diff --git a/datahub-web-react/src/app/entity/container/ContainerEntitiesTab.tsx b/datahub-web-react/src/app/entity/container/ContainerEntitiesTab.tsx index 45161a99a1067e..b4fd67806edfe1 100644 --- a/datahub-web-react/src/app/entity/container/ContainerEntitiesTab.tsx +++ b/datahub-web-react/src/app/entity/container/ContainerEntitiesTab.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { useEntityData } from '../shared/EntityContext'; -import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch'; +import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection'; export const ContainerEntitiesTab = () => { const { urn } = useEntityData(); @@ -11,7 +11,7 @@ export const ContainerEntitiesTab = () => { }; return ( - { subTypes={data.subTypes} container={data.container} entityCount={data.entities?.total} - domain={data.domain} + domain={data.domain?.domain} + tags={data.tags} /> ); }; @@ -133,13 +134,16 @@ export class ContainerEntity implements Entity { platformName={data.platform.properties?.displayName || data.platform.name} platformLogo={data.platform.properties?.logoUrl} platformInstanceId={data.dataPlatformInstance?.instanceId} - description={data.properties?.description} + description={data.editableProperties?.description || data.properties?.description} owners={data.ownership?.owners} subTypes={data.subTypes} container={data.container} entityCount={data.entities?.total} - domain={data.domain} + domain={data.domain?.domain} parentContainers={data.parentContainers} + externalUrl={data.properties?.externalUrl} + tags={data.tags} + glossaryTerms={data.glossaryTerms} /> ); }; @@ -162,4 +166,14 @@ export class ContainerEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/container/preview/Preview.tsx b/datahub-web-react/src/app/entity/container/preview/Preview.tsx index 3e90a7d65cf42e..ddfd101f01434c 100644 --- a/datahub-web-react/src/app/entity/container/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/container/preview/Preview.tsx @@ -1,4 +1,6 @@ +import { Typography } from 'antd'; import React from 'react'; +import styled from 'styled-components'; import { Container, EntityType, @@ -7,10 +9,18 @@ import { SubTypes, Domain, ParentContainersResult, + GlobalTags, + Deprecation, + GlossaryTerms, } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { IconStyleType } from '../../Entity'; +import { ANTD_GRAY } from '../../shared/constants'; + +const StatText = styled(Typography.Text)` + color: ${ANTD_GRAY[8]}; +`; export const Preview = ({ urn, @@ -20,6 +30,8 @@ export const Preview = ({ platformInstanceId, description, owners, + tags, + glossaryTerms, insights, subTypes, logoComponent, @@ -27,6 +39,8 @@ export const Preview = ({ entityCount, domain, parentContainers, + externalUrl, + deprecation, }: { urn: string; name: string; @@ -35,13 +49,17 @@ export const Preview = ({ platformInstanceId?: string; description?: string | null; owners?: Array | null; + tags?: GlobalTags | null; + glossaryTerms?: GlossaryTerms | null; insights?: Array | null; subTypes?: SubTypes | null; logoComponent?: JSX.Element; container?: Container | null; entityCount?: number; domain?: Domain | null; + deprecation?: Deprecation | null; parentContainers?: ParentContainersResult | null; + externalUrl?: string | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); const typeName = (subTypes?.typeNames?.length && subTypes?.typeNames[0]) || 'Container'; @@ -54,14 +72,25 @@ export const Preview = ({ description={description || ''} type={typeName} owners={owners} + deprecation={deprecation} insights={insights} logoUrl={platformLogo || undefined} logoComponent={logoComponent} container={container || undefined} typeIcon={entityRegistry.getIcon(EntityType.Container, 12, IconStyleType.ACCENT)} - entityCount={entityCount} domain={domain || undefined} parentContainers={parentContainers} + tags={tags || undefined} + glossaryTerms={glossaryTerms || undefined} + externalUrl={externalUrl} + subHeader={ + (entityCount && [ + + {entityCount} {entityCount === 1 ? 'entity' : 'entities'} + , + ]) || + undefined + } /> ); }; diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx index 81e99e03fdf57f..016cb64d7347a7 100644 --- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx +++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx @@ -5,21 +5,23 @@ import { useGetDashboardQuery, useUpdateDashboardMutation, } from '../../../graphql/dashboard.generated'; -import { Dashboard, EntityType, OwnershipType, PlatformType, SearchResult } from '../../../types.generated'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Dashboard, EntityType, OwnershipType, SearchResult } from '../../../types.generated'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/SidebarOwnerSection'; import { SidebarAboutSection } from '../shared/containers/profile/sidebar/SidebarAboutSection'; import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; import { DashboardChartsTab } from '../shared/tabs/Entity/DashboardChartsTab'; +import { DashboardDatasetsTab } from '../shared/tabs/Entity/DashboardDatasetsTab'; import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; import { GenericEntityProperties } from '../shared/types'; import { DashboardPreview } from './preview/DashboardPreview'; import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { capitalizeFirstLetter } from '../../shared/textUtil'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; +import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; +import { DashboardStatsSummarySubHeader } from './profile/DashboardStatsSummarySubHeader'; /** * Definition of the DataHub Dashboard entity. @@ -74,6 +76,9 @@ export class DashboardEntity implements Entity { useUpdateQuery={useUpdateDashboardMutation} getOverrideProperties={this.getOverridePropertiesFromEntity} headerDropdownItems={new Set([EntityMenuItems.COPY_URL, EntityMenuItems.UPDATE_DEPRECATION])} + subHeader={{ + component: DashboardStatsSummarySubHeader, + }} tabs={[ { name: 'Documentation', @@ -83,14 +88,37 @@ export class DashboardEntity implements Entity { name: 'Properties', component: PropertiesTab, }, + { + name: 'Lineage', + component: LineageTab, + display: { + visible: (_, _1) => true, + enabled: (_, dashboard: GetDashboardQuery) => { + return ( + (dashboard?.dashboard?.upstream?.total || 0) > 0 || + (dashboard?.dashboard?.downstream?.total || 0) > 0 + ); + }, + }, + }, { name: 'Charts', component: DashboardChartsTab, display: { - visible: (_, _1) => true, + visible: (_, dashboard: GetDashboardQuery) => + (dashboard?.dashboard?.charts?.total || 0) > 0 || + (dashboard?.dashboard?.datasets?.total || 0) === 0, enabled: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.charts?.total || 0) > 0, }, }, + { + name: 'Datasets', + component: DashboardDatasetsTab, + display: { + visible: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.datasets?.total || 0) > 0, + enabled: (_, dashboard: GetDashboardQuery) => (dashboard?.dashboard?.datasets?.total || 0) > 0, + }, + }, ]} sidebarSections={[ { @@ -118,23 +146,11 @@ export class DashboardEntity implements Entity { getOverridePropertiesFromEntity = (dashboard?: Dashboard | null): GenericEntityProperties => { // TODO: Get rid of this once we have correctly formed platform coming back. - const tool = dashboard?.tool || ''; const name = dashboard?.properties?.name; const externalUrl = dashboard?.properties?.externalUrl; return { name, externalUrl, - platform: { - urn: `urn:li:dataPlatform:(${tool})`, - type: EntityType.DataPlatform, - name: tool, - properties: { - logoUrl: dashboard?.platform?.properties?.logoUrl || '', - displayName: capitalizeFirstLetter(tool), - type: PlatformType.Others, - datasetNameDelimiter: '.', - }, - }, }; }; @@ -142,7 +158,7 @@ export class DashboardEntity implements Entity { return ( { owners={data.ownership?.owners} glossaryTerms={data?.glossaryTerms} logoUrl={data?.platform?.properties?.logoUrl} - domain={data.domain} + domain={data.domain?.domain} container={data.container} + parentContainers={data.parentContainers} + deprecation={data.deprecation} + externalUrl={data.properties?.externalUrl} + statsSummary={data.statsSummary} + lastUpdatedMs={data.properties?.lastModified?.time} + createdMs={data.properties?.created?.time} /> ); }; @@ -161,7 +183,7 @@ export class DashboardEntity implements Entity { return ( { glossaryTerms={data?.glossaryTerms} insights={result.insights} logoUrl={data?.platform?.properties?.logoUrl || ''} - domain={data.domain} + domain={data.domain?.domain} container={data.container} parentContainers={data.parentContainers} + deprecation={data.deprecation} + externalUrl={data.properties?.externalUrl} + statsSummary={data.statsSummary} + lastUpdatedMs={data.properties?.lastModified?.time} + createdMs={data.properties?.created?.time} /> ); }; @@ -184,7 +211,7 @@ export class DashboardEntity implements Entity { name: entity.properties?.name || '', type: EntityType.Dashboard, icon: entity?.platform?.properties?.logoUrl || '', - platform: entity.tool, + platform: entity?.platform.properties?.displayName || entity?.platform.name, }; }; @@ -199,4 +226,15 @@ export class DashboardEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx b/datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx index dd9076c43e0f1e..c04984edf91210 100644 --- a/datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx +++ b/datahub-web-react/src/app/entity/dashboard/preview/DashboardPreview.tsx @@ -9,10 +9,14 @@ import { Owner, SearchInsight, ParentContainersResult, + Deprecation, + DashboardStatsSummary, } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { capitalizeFirstLetter } from '../../../shared/textUtil'; +import { IconStyleType } from '../../Entity'; +import { DashboardStatsSummary as DashboardStatsSummaryView } from '../shared/DashboardStatsSummary'; export const DashboardPreview = ({ urn, @@ -28,7 +32,13 @@ export const DashboardPreview = ({ container, insights, logoUrl, + chartCount, + statsSummary, + lastUpdatedMs, + createdMs, + externalUrl, parentContainers, + deprecation, }: { urn: string; platform: string; @@ -41,8 +51,14 @@ export const DashboardPreview = ({ glossaryTerms?: GlossaryTerms | null; domain?: Domain | null; container?: Container | null; + deprecation?: Deprecation | null; insights?: Array | null; logoUrl?: string | null; + chartCount?: number | null; + statsSummary?: DashboardStatsSummary | null; + lastUpdatedMs?: number | null; + createdMs?: number | null; + externalUrl?: string | null; parentContainers?: ParentContainersResult | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); @@ -54,6 +70,7 @@ export const DashboardPreview = ({ name={name || ''} description={description || ''} type="Dashboard" + typeIcon={entityRegistry.getIcon(EntityType.Dashboard, 14, IconStyleType.ACCENT)} logoUrl={logoUrl || ''} platformInstanceId={platformInstanceId} platform={capitalizedPlatform} @@ -63,8 +80,20 @@ export const DashboardPreview = ({ container={container || undefined} glossaryTerms={glossaryTerms || undefined} domain={domain} + deprecation={deprecation} insights={insights} parentContainers={parentContainers} + externalUrl={externalUrl} + topUsers={statsSummary?.topUsersLast30Days} + subHeader={ + + } /> ); }; diff --git a/datahub-web-react/src/app/entity/dashboard/profile/DashboardStatsSummarySubHeader.tsx b/datahub-web-react/src/app/entity/dashboard/profile/DashboardStatsSummarySubHeader.tsx new file mode 100644 index 00000000000000..9ce1fbe16e83f2 --- /dev/null +++ b/datahub-web-react/src/app/entity/dashboard/profile/DashboardStatsSummarySubHeader.tsx @@ -0,0 +1,26 @@ +import React from 'react'; +import { DashboardStatsSummary as DashboardStatsSummaryObj } from '../../../../types.generated'; +import { useBaseEntity } from '../../shared/EntityContext'; +import { GetDashboardQuery } from '../../../../graphql/dashboard.generated'; +import { DashboardStatsSummary } from '../shared/DashboardStatsSummary'; + +export const DashboardStatsSummarySubHeader = () => { + const result = useBaseEntity(); + const dashboard = result?.dashboard; + const maybeStatsSummary = dashboard?.statsSummary as DashboardStatsSummaryObj; + const chartCount = dashboard?.charts?.total; + const viewCount = maybeStatsSummary?.viewCount; + const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days; + const lastUpdatedMs = dashboard?.properties?.lastModified?.time; + const createdMs = dashboard?.properties?.created?.time; + + return ( + + ); +}; diff --git a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx new file mode 100644 index 00000000000000..510c94efde68ea --- /dev/null +++ b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx @@ -0,0 +1,79 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Popover, Tooltip } from 'antd'; +import { ClockCircleOutlined, EyeOutlined, TeamOutlined, QuestionCircleOutlined } from '@ant-design/icons'; +import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber'; +import { ANTD_GRAY } from '../../shared/constants'; +import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils'; +import { StatsSummary } from '../../shared/components/styled/StatsSummary'; + +const StatText = styled.span` + color: ${ANTD_GRAY[8]}; +`; + +const HelpIcon = styled(QuestionCircleOutlined)` + color: ${ANTD_GRAY[7]}; + padding-left: 4px; +`; + +type Props = { + chartCount?: number | null; + viewCount?: number | null; + uniqueUserCountLast30Days?: number | null; + lastUpdatedMs?: number | null; + createdMs?: number | null; +}; + +export const DashboardStatsSummary = ({ + chartCount, + viewCount, + uniqueUserCountLast30Days, + lastUpdatedMs, + createdMs, +}: Props) => { + const statsViews = [ + (!!chartCount && ( + + {chartCount} charts + + )) || + undefined, + (!!viewCount && ( + + + {formatNumberWithoutAbbreviation(viewCount)} views + + )) || + undefined, + (!!uniqueUserCountLast30Days && ( + + + {formatNumberWithoutAbbreviation(uniqueUserCountLast30Days)} unique users + + )) || + undefined, + (!!lastUpdatedMs && ( + + {createdMs &&
Created on {toLocalDateTimeString(createdMs)}.
} +
+ Changed on {toLocalDateTimeString(lastUpdatedMs)}.{' '} + + + +
+ + } + > + + + Changed {toRelativeTimeString(lastUpdatedMs)} + + + )) || + undefined, + ].filter((stat) => stat !== undefined); + + return <>{statsViews.length > 0 && }; +}; diff --git a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx index 28166672122a6b..b148d1b25a1dd9 100644 --- a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx +++ b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx @@ -1,8 +1,8 @@ import * as React from 'react'; import { ShareAltOutlined } from '@ant-design/icons'; -import { DataFlow, EntityType, OwnershipType, PlatformType, SearchResult } from '../../../types.generated'; +import { DataFlow, EntityType, OwnershipType, SearchResult } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; import { useGetDataFlowQuery, useUpdateDataFlowMutation } from '../../../graphql/dataFlow.generated'; import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; @@ -13,7 +13,6 @@ import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Owners import { GenericEntityProperties } from '../shared/types'; import { DataFlowJobsTab } from '../shared/tabs/Entity/DataFlowJobsTab'; import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { capitalizeFirstLetter } from '../../shared/textUtil'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; @@ -104,57 +103,46 @@ export class DataFlowEntity implements Entity { getOverridePropertiesFromEntity = (dataFlow?: DataFlow | null): GenericEntityProperties => { // TODO: Get rid of this once we have correctly formed platform coming back. - const tool = dataFlow?.orchestrator || ''; const name = dataFlow?.properties?.name; const externalUrl = dataFlow?.properties?.externalUrl; return { name, externalUrl, - platform: { - urn: `urn:li:dataPlatform:(${tool})`, - type: EntityType.DataPlatform, - name: tool, - properties: { - logoUrl: dataFlow?.platform?.properties?.logoUrl || '', - displayName: capitalizeFirstLetter(tool), - type: PlatformType.Others, - datasetNameDelimiter: '.', - }, - }, }; }; renderPreview = (_: PreviewType, data: DataFlow) => { - const platformName = data.orchestrator.charAt(0).toUpperCase() + data.orchestrator.slice(1); return ( ); }; renderSearch = (result: SearchResult) => { const data = result.entity as DataFlow; - const platformName = data.orchestrator.charAt(0).toUpperCase() + data.orchestrator.slice(1); return ( ); }; @@ -170,4 +158,15 @@ export class DataFlowEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx index 12cc1823dcf0c4..a69ca18c660576 100644 --- a/datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx @@ -1,8 +1,16 @@ import React from 'react'; -import { Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated'; +import { Typography } from 'antd'; +import styled from 'styled-components'; +import { Deprecation, Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { capitalizeFirstLetter } from '../../../shared/textUtil'; +import { IconStyleType } from '../../Entity'; +import { ANTD_GRAY } from '../../shared/constants'; + +const StatText = styled(Typography.Text)` + color: ${ANTD_GRAY[8]}; +`; export const Preview = ({ urn, @@ -14,8 +22,11 @@ export const Preview = ({ owners, globalTags, domain, + externalUrl, snippet, insights, + jobCount, + deprecation, }: { urn: string; name: string; @@ -26,8 +37,11 @@ export const Preview = ({ owners?: Array | null; domain?: Domain | null; globalTags?: GlobalTags | null; + deprecation?: Deprecation | null; + externalUrl?: string | null; snippet?: React.ReactNode | null; insights?: Array | null; + jobCount?: number | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); const capitalizedPlatform = capitalizeFirstLetter(platformName); @@ -38,6 +52,7 @@ export const Preview = ({ description={description || ''} platformInstanceId={platformInstanceId} type="Data Pipeline" + typeIcon={entityRegistry.getIcon(EntityType.DataFlow, 14, IconStyleType.ACCENT)} platform={capitalizedPlatform} logoUrl={platformLogo || ''} owners={owners} @@ -45,6 +60,16 @@ export const Preview = ({ domain={domain} snippet={snippet} insights={insights} + externalUrl={externalUrl} + deprecation={deprecation} + subHeader={ + (jobCount && [ + + {jobCount} {entityRegistry.getCollectionName(EntityType.DataJob)} + , + ]) || + undefined + } /> ); }; diff --git a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx index a55da1a79307fc..6aeb4ccd5e4622 100644 --- a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx +++ b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx @@ -1,8 +1,8 @@ import * as React from 'react'; import { ConsoleSqlOutlined } from '@ant-design/icons'; -import { DataJob, EntityType, OwnershipType, PlatformType, SearchResult } from '../../../types.generated'; +import { DataJob, EntityType, OwnershipType, SearchResult } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; import { GetDataJobQuery, useGetDataJobQuery, useUpdateDataJobMutation } from '../../../graphql/dataJob.generated'; import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; @@ -14,11 +14,14 @@ import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Owners import { GenericEntityProperties } from '../shared/types'; import { DataJobFlowTab } from '../shared/tabs/Entity/DataJobFlowTab'; import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { capitalizeFirstLetter } from '../../shared/textUtil'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; import { RunsTab } from './tabs/RunsTab'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; +const getDataJobPlatformName = (data?: DataJob): string => { + return data?.dataFlow?.platform?.properties?.displayName || data?.dataFlow?.platform?.name || ''; +}; + /** * Definition of the DataHub DataJob entity. */ @@ -85,8 +88,8 @@ export class DataJobEntity implements Entity { display: { visible: (_, _1) => true, enabled: (_, dataJob: GetDataJobQuery) => - (dataJob?.dataJob?.incoming?.count || 0) !== 0 || - (dataJob?.dataJob?.outgoing?.count || 0) !== 0, + (dataJob?.dataJob?.upstream?.count || 0) !== 0 || + (dataJob?.dataJob?.downstream?.count || 0) !== 0, }, }, { @@ -124,61 +127,49 @@ export class DataJobEntity implements Entity { getOverridePropertiesFromEntity = (dataJob?: DataJob | null): GenericEntityProperties => { // TODO: Get rid of this once we have correctly formed platform coming back. - const tool = dataJob?.dataFlow?.orchestrator || ''; const name = dataJob?.properties?.name; const externalUrl = dataJob?.properties?.externalUrl; return { name, externalUrl, - platform: { - urn: `urn:li:dataPlatform:(${tool})`, - type: EntityType.DataPlatform, - name: tool, - properties: { - logoUrl: dataJob?.dataFlow?.platform?.properties?.logoUrl || '', - displayName: capitalizeFirstLetter(tool), - type: PlatformType.Others, - datasetNameDelimiter: '.', - }, - }, + platform: dataJob?.dataFlow?.platform, }; }; renderPreview = (_: PreviewType, data: DataJob) => { - const platformName = data.dataFlow - ? data.dataFlow?.orchestrator.charAt(0).toUpperCase() + data.dataFlow?.orchestrator.slice(1) - : ''; return ( ); }; renderSearch = (result: SearchResult) => { const data = result.entity as DataJob; - const platformName = data.dataFlow - ? data.dataFlow?.orchestrator.charAt(0).toUpperCase() + data.dataFlow?.orchestrator.slice(1) - : ''; return ( ); }; @@ -189,7 +180,7 @@ export class DataJobEntity implements Entity { name: entity?.properties?.name || '', type: EntityType.DataJob, icon: entity?.dataFlow?.platform?.properties?.logoUrl || '', - platform: entity?.dataFlow?.orchestrator || '', + platform: getDataJobPlatformName(entity), }; }; @@ -204,4 +195,15 @@ export class DataJobEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx index 5bc138cc5f455c..e61d8ad0c9fe66 100644 --- a/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/dataJob/preview/Preview.tsx @@ -1,8 +1,19 @@ import React from 'react'; -import { Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated'; +import styled from 'styled-components'; +import { Typography } from 'antd'; +import { ClockCircleOutlined } from '@ant-design/icons'; + +import { Deprecation, Domain, EntityType, GlobalTags, Owner, SearchInsight } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { capitalizeFirstLetter } from '../../../shared/textUtil'; +import { IconStyleType } from '../../Entity'; +import { ANTD_GRAY } from '../../shared/constants'; +import { toRelativeTimeString } from '../../../shared/time/timeUtils'; + +const StatText = styled(Typography.Text)` + color: ${ANTD_GRAY[8]}; +`; export const Preview = ({ urn, @@ -13,9 +24,12 @@ export const Preview = ({ platformInstanceId, owners, domain, + deprecation, globalTags, snippet, insights, + lastRunTimeMs, + externalUrl, }: { urn: string; name: string; @@ -25,9 +39,12 @@ export const Preview = ({ platformInstanceId?: string; owners?: Array | null; domain?: Domain | null; + deprecation?: Deprecation | null; globalTags?: GlobalTags | null; snippet?: React.ReactNode | null; insights?: Array | null; + lastRunTimeMs?: number | null; + externalUrl?: string | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); const capitalizedPlatform = capitalizeFirstLetter(platformName); @@ -37,6 +54,7 @@ export const Preview = ({ name={name} description={description || ''} type="Data Task" + typeIcon={entityRegistry.getIcon(EntityType.DataJob, 14, IconStyleType.ACCENT)} platform={capitalizedPlatform} logoUrl={platformLogo || ''} platformInstanceId={platformInstanceId} @@ -44,8 +62,19 @@ export const Preview = ({ tags={globalTags || undefined} domain={domain} snippet={snippet} + deprecation={deprecation} dataTestID="datajob-item-preview" insights={insights} + externalUrl={externalUrl} + subHeader={ + (lastRunTimeMs && [ + + + Last run {toRelativeTimeString(lastRunTimeMs)} + , + ]) || + undefined + } /> ); }; diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index 602d87fbf93aee..bf0e8202346fab 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { DatabaseFilled, DatabaseOutlined } from '@ant-design/icons'; import { Typography } from 'antd'; import { Dataset, DatasetProperties, EntityType, OwnershipType, SearchResult } from '../../../types.generated'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { Preview } from './preview/Preview'; import { FIELDS_TO_HIGHLIGHT } from './search/highlights'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; @@ -15,7 +15,6 @@ import QueriesTab from '../shared/tabs/Dataset/Queries/QueriesTab'; import { SidebarAboutSection } from '../shared/containers/profile/sidebar/SidebarAboutSection'; import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/SidebarOwnerSection'; import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; -import { SidebarStatsSection } from '../shared/containers/profile/sidebar/Dataset/StatsSidebarSection'; import StatsTab from '../shared/tabs/Dataset/Stats/StatsTab'; import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; import { capitalizeFirstLetter } from '../../shared/textUtil'; @@ -32,6 +31,8 @@ import { EditSampleTab } from '../shared/tabs/Dataset/Schema/EditSampleTab'; import { ValidationsTab } from '../shared/tabs/Dataset/Validations/ValidationsTab'; import { OperationsTab } from './profile/OperationsTab'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; +import { SidebarSiblingsSection } from '../shared/containers/profile/sidebar/SidebarSiblingsSection'; +import { DatasetStatsSummarySubHeader } from './profile/stats/stats/DatasetStatsSummarySubHeader'; const SUBTYPES = { VIEW: 'view', @@ -90,6 +91,9 @@ export class DatasetEntity implements Entity { useUpdateQuery={useUpdateDatasetMutation} getOverrideProperties={this.getOverridePropertiesFromEntity} headerDropdownItems={new Set([EntityMenuItems.COPY_URL, EntityMenuItems.UPDATE_DEPRECATION])} + subHeader={{ + component: DatasetStatsSummarySubHeader, + }} tabs={[ { name: 'Schema', @@ -236,18 +240,23 @@ export class DatasetEntity implements Entity { component: SidebarAboutSection, }, { - component: SidebarViewDefinitionSection, + component: SidebarOwnerSection, + properties: { + defaultOwnerType: OwnershipType.TechnicalOwner, + }, + }, + { + component: SidebarSiblingsSection, display: { visible: (_, dataset: GetDatasetQuery) => - (dataset?.dataset?.viewProperties?.logic && true) || false, + (dataset?.dataset?.siblings?.siblings?.length || 0) > 0, }, }, { - component: SidebarStatsSection, + component: SidebarViewDefinitionSection, display: { visible: (_, dataset: GetDatasetQuery) => - (dataset?.dataset?.datasetProfiles?.length || 0) > 0 || - (dataset?.dataset?.usageStats?.buckets?.length || 0) > 0, + (dataset?.dataset?.viewProperties?.logic && true) || false, }, }, { @@ -257,12 +266,6 @@ export class DatasetEntity implements Entity { hasTerms: true, }, }, - { - component: SidebarOwnerSection, - properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, - }, - }, { component: SidebarDomainSection, }, @@ -302,7 +305,7 @@ export class DatasetEntity implements Entity { owners={data.ownership?.owners} globalTags={data.globalTags} glossaryTerms={data.glossaryTerms} - domain={data.domain} + domain={data.domain?.domain} container={data.container} /> ); @@ -310,6 +313,7 @@ export class DatasetEntity implements Entity { renderSearch = (result: SearchResult) => { const data = result.entity as Dataset; + const genericProperties = this.getGenericEntityProperties(data); return ( { platformName={data.platform.properties?.displayName || data.platform.name} platformLogo={data.platform.properties?.logoUrl} platformInstanceId={data.dataPlatformInstance?.instanceId} + platformNames={genericProperties?.siblingPlatforms?.map( + (platform) => platform.properties?.displayName || platform.name, + )} + platformLogos={genericProperties?.siblingPlatforms?.map((platform) => platform.properties?.logoUrl)} owners={data.ownership?.owners} globalTags={data.globalTags} - domain={data.domain} + domain={data.domain?.domain} + deprecation={data.deprecation} glossaryTerms={data.glossaryTerms} subtype={data.subTypes?.typeNames?.[0]} container={data.container} @@ -337,6 +346,12 @@ export class DatasetEntity implements Entity { ) } insights={result.insights} + externalUrl={data.properties?.externalUrl} + statsSummary={data.statsSummary} + rowCount={(data as any).lastProfile?.length && (data as any).lastProfile[0].rowCount} + lastUpdatedMs={ + (data as any).lastOperation?.length && (data as any).lastOperation[0].lastUpdatedTimestamp + } /> ); }; @@ -354,7 +369,7 @@ export class DatasetEntity implements Entity { }; displayName = (data: Dataset) => { - return data?.properties?.name || data.name; + return data?.properties?.name || data.name || data.urn; }; platformLogoUrl = (data: Dataset) => { @@ -368,4 +383,15 @@ export class DatasetEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx index e6e64be6a12ee4..8e6f1a8b660bc8 100644 --- a/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/dataset/preview/Preview.tsx @@ -9,11 +9,15 @@ import { Domain, Container, ParentContainersResult, + Maybe, + Deprecation, + DatasetStatsSummary, } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { capitalizeFirstLetterOnly } from '../../../shared/textUtil'; import { IconStyleType } from '../../Entity'; +import { DatasetStatsSummary as DatasetStatsSummaryView } from '../shared/DatasetStatsSummary'; export const Preview = ({ urn, @@ -22,16 +26,23 @@ export const Preview = ({ description, platformName, platformLogo, + platformNames, + platformLogos, platformInstanceId, owners, globalTags, domain, + deprecation, snippet, insights, glossaryTerms, subtype, + externalUrl, container, parentContainers, + rowCount, + statsSummary, + lastUpdatedMs, }: { urn: string; name: string; @@ -39,16 +50,23 @@ export const Preview = ({ description?: string | null; platformName: string; platformLogo?: string | null; + platformNames?: (Maybe | undefined)[]; + platformLogos?: (Maybe | undefined)[]; platformInstanceId?: string; owners?: Array | null; domain?: Domain | null; + deprecation?: Deprecation | null; globalTags?: GlobalTags | null; snippet?: React.ReactNode | null; insights?: Array | null; glossaryTerms?: GlossaryTerms | null; subtype?: string | null; + externalUrl?: string | null; container?: Container | null; parentContainers?: ParentContainersResult | null; + rowCount?: number | null; + statsSummary?: DatasetStatsSummary | null; + lastUpdatedMs?: number | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); const capitalPlatformName = capitalizeFirstLetterOnly(platformName); @@ -61,16 +79,29 @@ export const Preview = ({ logoUrl={platformLogo || ''} typeIcon={entityRegistry.getIcon(EntityType.Dataset, 12, IconStyleType.ACCENT)} platform={capitalPlatformName} + platforms={platformNames} + logoUrls={platformLogos} platformInstanceId={platformInstanceId} qualifier={origin} tags={globalTags || undefined} owners={owners} domain={domain} container={container || undefined} + deprecation={deprecation} snippet={snippet} glossaryTerms={glossaryTerms || undefined} insights={insights} parentContainers={parentContainers} + externalUrl={externalUrl} + topUsers={statsSummary?.topUsersLast30Days} + subHeader={ + + } /> ); }; diff --git a/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx b/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx new file mode 100644 index 00000000000000..c8bb5d8100f2aa --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/SchemaTimeStamps.test.tsx @@ -0,0 +1,23 @@ +import { render } from '@testing-library/react'; +import React from 'react'; +import { toRelativeTimeString } from '../../../../../shared/time/timeUtils'; +import SchemaTimeStamps from '../../schema/components/SchemaTimeStamps'; + +describe('SchemaTimeStamps', () => { + it('should render last observed text if lastObserved is not null', () => { + const { getByText, queryByText } = render(); + expect(getByText(`Last observed ${toRelativeTimeString(123)}`)).toBeInTheDocument(); + expect(queryByText(`Reported ${toRelativeTimeString(123)}`)).toBeNull(); + }); + + it('should render last updated text if lastObserved is null', () => { + const { getByText, queryByText } = render(); + expect(queryByText(`Last observed ${toRelativeTimeString(123)}`)).toBeNull(); + expect(getByText(`Reported ${toRelativeTimeString(123)}`)).toBeInTheDocument(); + }); + + it('should return null if lastUpdated and lastObserved are both null', () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); +}); diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx index 2644dcffa77f03..52e9dbac660f97 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx @@ -1,15 +1,21 @@ import React from 'react'; import { useHistory, useLocation } from 'react-router-dom'; -import { Button, Popover, Radio, Select, Typography } from 'antd'; -import { CaretDownOutlined, FileTextOutlined, InfoCircleOutlined, TableOutlined } from '@ant-design/icons'; +import { Button, Popover, Select, Tooltip, Typography } from 'antd'; +import { + AuditOutlined, + CaretDownOutlined, + FileTextOutlined, + QuestionCircleOutlined, + TableOutlined, +} from '@ant-design/icons'; import styled from 'styled-components'; import CustomPagination from './CustomPagination'; import TabToolbar from '../../../../shared/components/styled/TabToolbar'; import { SemanticVersionStruct } from '../../../../../../types.generated'; import { toRelativeTimeString } from '../../../../../shared/time/timeUtils'; -import { SchemaViewType } from '../utils/types'; -import { ANTD_GRAY } from '../../../../shared/constants'; +import { ANTD_GRAY, REDESIGN_COLORS } from '../../../../shared/constants'; import { navigateToVersionedDatasetUrl } from '../../../../shared/tabs/Dataset/Schema/utils/navigateToVersionedDatasetUrl'; +import SchemaTimeStamps from './SchemaTimeStamps'; const SchemaHeaderContainer = styled.div` display: flex; @@ -85,34 +91,16 @@ const SchemaBlameSelectorOption = styled(Select.Option)` } `; -const BlameRadio = styled(Radio.Group)` +const SchemaAuditButton = styled(Button)` &&& { margin-top: 6px; - margin-right: 10px; - min-width: 140px; - } -`; - -const BlameRadioButton = styled(Radio.Button)` - &&& { - min-width: 30px; - } -`; - -const CurrentVersionTimestampText = styled(Typography.Text)` - &&& { - line-height: 22px; - margin-top: 10px; - margin-right: 10px; - color: ${ANTD_GRAY[7]}; - min-width: 220px; } `; -const StyledInfoCircleOutlined = styled(InfoCircleOutlined)` +const StyledQuestionCircleOutlined = styled(QuestionCircleOutlined)` &&& { - margin-top: 12px; - font-size: 20px; + margin-top: 14px; + font-size: 16px; color: ${ANTD_GRAY[6]}; } `; @@ -134,11 +122,12 @@ type Props = { hasKeySchema: boolean; showKeySchema: boolean; setShowKeySchema: (show: boolean) => void; - lastUpdatedTimeString: string; + lastUpdated?: number | null; + lastObserved?: number | null; selectedVersion: string; versionList: Array; - schemaView: SchemaViewType; - setSchemaView: any; + showSchemaAuditView: boolean; + setShowSchemaAuditView: any; }; export default function SchemaHeader({ @@ -152,11 +141,12 @@ export default function SchemaHeader({ hasKeySchema, showKeySchema, setShowKeySchema, - lastUpdatedTimeString, + lastUpdated, + lastObserved, selectedVersion, versionList, - schemaView, - setSchemaView, + showSchemaAuditView, + setShowSchemaAuditView, }: Props) { const history = useHistory(); const location = useLocation(); @@ -174,6 +164,7 @@ export default function SchemaHeader({ 'unknown'; return `${semanticVersion.semanticVersion} - ${semanticVersionTimestampString}`; }; + const numVersions = versionList.length; const renderOptions = () => { return versionList.map( @@ -189,10 +180,7 @@ export default function SchemaHeader({ ), ); }; - - const onSchemaViewToggle = (e) => { - setSchemaView(e.target.value); - }; + const schemaAuditToggleText = showSchemaAuditView ? 'Close column history' : 'View column history'; const docLink = 'https://datahubproject.io/docs/dev-guides/timeline/'; return ( @@ -233,46 +221,52 @@ export default function SchemaHeader({ ))} - {lastUpdatedTimeString} - - - Normal - - - Blame - - - { - const datasetVersion: string = e as string; - navigateToVersionedDatasetUrl({ - location, - history, - datasetVersion, - }); - }} - data-testid="schema-version-selector-dropdown" - suffixIcon={} - > - {renderOptions()} - - - Semantic versions for this view were computed using Technical Schema. You can find more - info about how we compute versions - - {' '} - here.{' '} - - - } - > - - + + + setShowSchemaAuditView(!showSchemaAuditView)} + style={{ color: showSchemaAuditView ? REDESIGN_COLORS.BLUE : ANTD_GRAY[7] }} + > + + + + {numVersions > 1 && ( + <> + { + const datasetVersion: string = e as string; + navigateToVersionedDatasetUrl({ + location, + history, + datasetVersion, + }); + }} + data-testid="schema-version-selector-dropdown" + suffixIcon={} + > + {renderOptions()} + + + Semantic versions for this view were computed using Technical Schema. You can + find more info about how DataHub computes versions + + {' '} + here.{' '} + + + } + > + + + + )} diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx new file mode 100644 index 00000000000000..42eae019b947a1 --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaTimeStamps.tsx @@ -0,0 +1,64 @@ +import { ClockCircleOutlined } from '@ant-design/icons'; +import { Popover, Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components/macro'; +import { toLocalDateTimeString, toRelativeTimeString } from '../../../../../shared/time/timeUtils'; +import { ANTD_GRAY } from '../../../../shared/constants'; + +const CurrentVersionTimestampText = styled(Typography.Text)` + &&& { + line-height: 22px; + margin-top: 10px; + margin-right: 10px; + color: ${ANTD_GRAY[7]}; + width: max-content; + } +`; + +const TimeStampWrapper = styled.div` + margin-bottom: 5px; +`; + +const StyledClockIcon = styled(ClockCircleOutlined)` + margin-right: 5px; +`; + +interface Props { + lastUpdated?: number | null; + lastObserved?: number | null; +} + +function SchemaTimeStamps(props: Props) { + const { lastUpdated, lastObserved } = props; + + if (!lastUpdated && !lastObserved) return null; + + return ( + + {lastObserved && ( + Last observed on {toLocalDateTimeString(lastObserved)}. + )} + {lastUpdated &&
First reported on {toLocalDateTimeString(lastUpdated)}.
} + + } + > + + {lastObserved && ( + + Last observed {toRelativeTimeString(lastObserved)} + + )} + {!lastObserved && lastUpdated && ( + + + Reported {toRelativeTimeString(lastUpdated)} + + )} + + + ); +} + +export default SchemaTimeStamps; diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx index 1e881ce83dade1..309455c0c4ff96 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx @@ -37,29 +37,24 @@ export default function useSchemaTitleRenderer( return (fieldPath: string, record: ExtendedSchemaFields): JSX.Element => { const fieldPathWithoutAnnotations = translateFieldPath(fieldPath); + const parentPathWithoutAnnotations = translateFieldPath(record.parent?.fieldPath || ''); + let pathToDisplay = fieldPathWithoutAnnotations; - const isOverflow = fieldPathWithoutAnnotations.length > MAX_FIELD_PATH_LENGTH; - - let [firstPath, lastPath] = fieldPathWithoutAnnotations.split(/\.(?=[^.]+$)/); + // if the parent path is a prefix of the field path, remove it for display purposes + if (parentPathWithoutAnnotations && fieldPathWithoutAnnotations.indexOf(parentPathWithoutAnnotations) === 0) { + // parent length + 1 because of the trailing `.` of the parent + pathToDisplay = fieldPathWithoutAnnotations.slice(parentPathWithoutAnnotations.length + 1); + } - if (isOverflow) { - if (lastPath.length >= MAX_FIELD_PATH_LENGTH) { - lastPath = `..${lastPath.substring(lastPath.length - MAX_FIELD_PATH_LENGTH)}`; - firstPath = ''; - } else { - firstPath = firstPath.substring(fieldPath.length - MAX_FIELD_PATH_LENGTH); - if (firstPath.includes('.')) { - firstPath = `..${firstPath.substring(firstPath.indexOf('.'))}`; - } else { - firstPath = '..'; - } - } + // if the field path is too long, truncate it + if (pathToDisplay.length > MAX_FIELD_PATH_LENGTH) { + pathToDisplay = `..${pathToDisplay.substring(pathToDisplay.length - MAX_FIELD_PATH_LENGTH)}`; } return ( <> - {lastPath || firstPath} + {pathToDisplay} {(schemaMetadata?.primaryKeys?.includes(fieldPath) || record.isPartOfKey) && } {schemaMetadata?.foreignKeys diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts index 3ed70e8e3d95ea..b71b21112ddb58 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/types.ts @@ -7,6 +7,7 @@ export interface ExtendedSchemaFields extends SchemaField { pastGlobalTags?: GlobalTags | null; isNewRow?: boolean; isDeletedRow?: boolean; + parent?: ExtendedSchemaFields; } export enum SchemaViewType { diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts index 732f1d27e86b63..51cb8ab0245196 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts @@ -121,6 +121,7 @@ export function groupByFieldPath( // if the parent field exists in the ouput, add the current row as a child if (parentRow) { row.depth = (parentRow.depth || 0) + 1; + row.parent = parentRow; parentRow.children = [...(parentRow.children || []), row]; } else { outputRows.push(row); diff --git a/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx b/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx new file mode 100644 index 00000000000000..655779ecac556d --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/profile/stats/stats/DatasetStatsSummarySubHeader.tsx @@ -0,0 +1,34 @@ +import React from 'react'; +import { + DatasetStatsSummary as DatasetStatsSummaryObj, + DatasetProfile, + Operation, +} from '../../../../../../types.generated'; +import { useBaseEntity } from '../../../../shared/EntityContext'; +import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; +import { DatasetStatsSummary } from '../../../shared/DatasetStatsSummary'; + +export const DatasetStatsSummarySubHeader = () => { + const result = useBaseEntity(); + const dataset = result?.dataset; + + const maybeStatsSummary = dataset?.statsSummary as DatasetStatsSummaryObj; + const maybeLastProfile = + ((dataset?.datasetProfiles?.length || 0) > 0 && (dataset?.datasetProfiles![0] as DatasetProfile)) || undefined; + const maybeLastOperation = + ((dataset?.operations?.length || 0) > 0 && (dataset?.operations![0] as Operation)) || undefined; + + const rowCount = maybeLastProfile?.rowCount; + const queryCountLast30Days = maybeStatsSummary?.queryCountLast30Days; + const uniqueUserCountLast30Days = maybeStatsSummary?.uniqueUserCountLast30Days; + const lastUpdatedMs = maybeLastOperation?.lastUpdatedTimestamp; + + return ( + + ); +}; diff --git a/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts b/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts index 71c27ab7bd2ce0..819ee2f9b58f1b 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts +++ b/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts @@ -113,6 +113,7 @@ export const sampleSchemaWithTags: Schema = { description: 'this is a legacy dataset', type: EntityType.Tag, }, + associatedUrn: 'mock:urn', }, ], }, @@ -131,6 +132,7 @@ export const sampleSchemaWithTags: Schema = { termSource: 'sample term source', }, }, + associatedUrn: 'mock:urn', }, ], }, @@ -237,6 +239,7 @@ export const sampleSchemaWithPkFk: SchemaMetadata = { description: 'this is a legacy dataset', type: EntityType.Tag, }, + associatedUrn: 'mock:urn', }, ], }, @@ -255,6 +258,7 @@ export const sampleSchemaWithPkFk: SchemaMetadata = { termSource: 'sample term source', }, }, + associatedUrn: 'mock:urn', }, ], }, diff --git a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx new file mode 100644 index 00000000000000..7e18bb6d88ac51 --- /dev/null +++ b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx @@ -0,0 +1,81 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Popover, Tooltip } from 'antd'; +import { + ClockCircleOutlined, + ConsoleSqlOutlined, + TableOutlined, + TeamOutlined, + QuestionCircleOutlined, +} from '@ant-design/icons'; +import { formatNumberWithoutAbbreviation } from '../../../shared/formatNumber'; +import { ANTD_GRAY } from '../../shared/constants'; +import { toLocalDateTimeString, toRelativeTimeString } from '../../../shared/time/timeUtils'; +import { StatsSummary } from '../../shared/components/styled/StatsSummary'; + +const StatText = styled.span` + color: ${ANTD_GRAY[8]}; +`; + +const HelpIcon = styled(QuestionCircleOutlined)` + color: ${ANTD_GRAY[7]}; + padding-left: 4px; +`; + +type Props = { + rowCount?: number | null; + queryCountLast30Days?: number | null; + uniqueUserCountLast30Days?: number | null; + lastUpdatedMs?: number | null; +}; + +export const DatasetStatsSummary = ({ + rowCount, + queryCountLast30Days, + uniqueUserCountLast30Days, + lastUpdatedMs, +}: Props) => { + const statsViews = [ + (!!rowCount && ( + + + {formatNumberWithoutAbbreviation(rowCount)} rows + + )) || + undefined, + (!!queryCountLast30Days && ( + + + {formatNumberWithoutAbbreviation(queryCountLast30Days)} queries last month + + )) || + undefined, + (!!uniqueUserCountLast30Days && ( + + + {formatNumberWithoutAbbreviation(uniqueUserCountLast30Days)} unique users + + )) || + undefined, + (!!lastUpdatedMs && ( + + Changed on {toLocalDateTimeString(lastUpdatedMs)}.{' '} + + + + + } + > + + + Changed {toRelativeTimeString(lastUpdatedMs)} + + + )) || + undefined, + ].filter((stat) => stat !== undefined); + + return <>{statsViews.length > 0 && }; +}; diff --git a/datahub-web-react/src/app/entity/domain/DomainEntitiesTab.tsx b/datahub-web-react/src/app/entity/domain/DomainEntitiesTab.tsx index f37c341c27aecd..a5db4ecac75e24 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntitiesTab.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntitiesTab.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { useEntityData } from '../shared/EntityContext'; import { EntityType } from '../../../types.generated'; -import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch'; +import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection'; export const DomainEntitiesTab = () => { const { urn, entityType } = useEntityData(); @@ -16,7 +16,7 @@ export const DomainEntitiesTab = () => { } return ( - { useEntityQuery={useGetDomainQuery} useUpdateQuery={undefined} getOverrideProperties={this.getOverridePropertiesFromEntity} - headerDropdownItems={new Set([EntityMenuItems.COPY_URL])} + headerDropdownItems={new Set([EntityMenuItems.COPY_URL, EntityMenuItems.DELETE])} + headerActionItems={new Set([EntityActionItem.BATCH_ADD_DOMAIN])} + isNameEditable tabs={[ { name: 'Entities', @@ -117,7 +121,7 @@ export class DomainEntity implements Entity { }; displayName = (data: Domain) => { - return data?.properties?.name || data?.id; + return data?.properties?.name || data?.id || data.urn; }; getOverridePropertiesFromEntity = (data: Domain) => { @@ -133,4 +137,9 @@ export class DomainEntity implements Entity { getOverrideProperties: this.getOverridePropertiesFromEntity, }); }; + + supportedCapabilities = () => { + // TODO.. Determine whether SOFT_DELETE should go into here. + return new Set([EntityCapabilityType.OWNERS]); + }; } diff --git a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx index ae021c83bf977d..d72518a57e91aa 100644 --- a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { EntityType, Owner, SearchInsight } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import { IconStyleType } from '../../Entity'; export const Preview = ({ urn, @@ -27,6 +28,7 @@ export const Preview = ({ name={name || ''} description={description || ''} type="Domain" + typeIcon={entityRegistry.getIcon(EntityType.Domain, 14, IconStyleType.ACCENT)} owners={owners} insights={insights} logoComponent={logoComponent} diff --git a/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx b/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx index 6030a8698bea43..9aa4f40c2ca2c4 100644 --- a/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx +++ b/datahub-web-react/src/app/entity/glossaryNode/ChildrenTab.tsx @@ -1,16 +1,22 @@ import React from 'react'; import { EntityType, GlossaryNode, GlossaryTerm } from '../../../types.generated'; import GlossaryEntitiesList from '../../glossary/GlossaryEntitiesList'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { sortGlossaryTerms } from '../glossaryTerm/utils'; import { useEntityData } from '../shared/EntityContext'; +import { sortGlossaryNodes } from './utils'; function ChildrenTab() { const { entityData } = useEntityData(); + const entityRegistry = useEntityRegistry(); const childNodes = entityData?.children?.relationships .filter((child) => child.entity?.type === EntityType.GlossaryNode) + .sort((nodeA, nodeB) => sortGlossaryNodes(entityRegistry, nodeA.entity, nodeB.entity)) .map((child) => child.entity); const childTerms = entityData?.children?.relationships .filter((child) => child.entity?.type === EntityType.GlossaryTerm) + .sort((termA, termB) => sortGlossaryTerms(entityRegistry, termA.entity, termB.entity)) .map((child) => child.entity); return ( diff --git a/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx b/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx index 137109cff26ef6..5075d262153c39 100644 --- a/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx +++ b/datahub-web-react/src/app/entity/glossaryNode/GlossaryNodeEntity.tsx @@ -3,7 +3,7 @@ import React from 'react'; import { useGetGlossaryNodeQuery } from '../../../graphql/glossaryNode.generated'; import { EntityType, GlossaryNode, SearchResult } from '../../../types.generated'; import GlossaryEntitiesPath from '../../glossary/GlossaryEntitiesPath'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/SidebarOwnerSection'; import { SidebarAboutSection } from '../shared/containers/profile/sidebar/SidebarAboutSection'; @@ -135,6 +135,14 @@ class GlossaryNodeEntity implements Entity { getOverrideProperties: (data) => data, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } export default GlossaryNodeEntity; diff --git a/datahub-web-react/src/app/entity/glossaryNode/utils.ts b/datahub-web-react/src/app/entity/glossaryNode/utils.ts new file mode 100644 index 00000000000000..4cf214e810f780 --- /dev/null +++ b/datahub-web-react/src/app/entity/glossaryNode/utils.ts @@ -0,0 +1,8 @@ +import { Entity, EntityType } from '../../../types.generated'; +import EntityRegistry from '../EntityRegistry'; + +export function sortGlossaryNodes(entityRegistry: EntityRegistry, nodeA?: Entity | null, nodeB?: Entity | null) { + const nodeAName = entityRegistry.getDisplayName(EntityType.GlossaryNode, nodeA); + const nodeBName = entityRegistry.getDisplayName(EntityType.GlossaryNode, nodeB); + return nodeAName.localeCompare(nodeBName); +} diff --git a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx index 5c9e9fc8a92c73..234d1d1aa6e467 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx @@ -1,7 +1,7 @@ import * as React from 'react'; import { BookFilled, BookOutlined } from '@ant-design/icons'; import { EntityType, GlossaryTerm, SearchResult } from '../../../types.generated'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { Preview } from './preview/Preview'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; @@ -16,6 +16,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab' import { SidebarAboutSection } from '../shared/containers/profile/sidebar/SidebarAboutSection'; import GlossaryEntitiesPath from '../../glossary/GlossaryEntitiesPath'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; +import { EntityActionItem } from '../shared/entity/EntityActions'; /** * Definition of the DataHub Dataset entity. @@ -62,6 +63,7 @@ export class GlossaryTermEntity implements Entity { urn={urn} entityType={EntityType.GlossaryTerm} useEntityQuery={useGetGlossaryTermQuery as any} + headerActionItems={new Set([EntityActionItem.BATCH_ADD_GLOSSARY_TERM])} headerDropdownItems={ new Set([ EntityMenuItems.COPY_URL, @@ -154,4 +156,12 @@ export class GlossaryTermEntity implements Entity { getOverrideProperties: (data) => data, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx b/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx index 6c11193817f20b..835054d6692295 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx @@ -1,19 +1,22 @@ import React from 'react'; import { BookOutlined } from '@ant-design/icons'; -import { EntityType, Owner } from '../../../../types.generated'; +import { Deprecation, EntityType, Owner } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import { IconStyleType } from '../../Entity'; export const Preview = ({ urn, name, description, owners, + deprecation, }: { urn: string; name: string; description?: string | null; owners?: Array | null; + deprecation?: Deprecation | null; }): JSX.Element => { const entityRegistry = useEntityRegistry(); return ( @@ -24,6 +27,8 @@ export const Preview = ({ owners={owners} logoComponent={} type="Glossary Term" + typeIcon={entityRegistry.getIcon(EntityType.GlossaryTerm, 14, IconStyleType.ACCENT)} + deprecation={deprecation} /> ); }; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx b/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx new file mode 100644 index 00000000000000..5b303f75e2985a --- /dev/null +++ b/datahub-web-react/src/app/entity/glossaryTerm/profile/AddRelatedTermsModal.tsx @@ -0,0 +1,205 @@ +import { message, Button, Modal, Select, Tag } from 'antd'; +import React, { useState } from 'react'; +import styled from 'styled-components/macro'; +import { useAddRelatedTermsMutation } from '../../../../graphql/glossaryTerm.generated'; +import { useGetSearchResultsLazyQuery } from '../../../../graphql/search.generated'; +import { EntityType, SearchResult, TermRelationshipType } from '../../../../types.generated'; +import GlossaryBrowser from '../../../glossary/GlossaryBrowser/GlossaryBrowser'; +import ClickOutside from '../../../shared/ClickOutside'; +import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; +import TermLabel from '../../../shared/TermLabel'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { useEntityData, useRefetch } from '../../shared/EntityContext'; + +const StyledSelect = styled(Select)` + width: 480px; +`; + +interface Props { + onClose: () => void; + relationshipType: TermRelationshipType; +} + +function AddRelatedTermsModal(props: Props) { + const { onClose, relationshipType } = props; + + const [inputValue, setInputValue] = useState(''); + const [selectedUrns, setSelectedUrns] = useState([]); + const [selectedTerms, setSelectedTerms] = useState([]); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); + const entityRegistry = useEntityRegistry(); + const { urn: entityDataUrn } = useEntityData(); + const refetch = useRefetch(); + + const [AddRelatedTerms] = useAddRelatedTermsMutation(); + + function addTerms() { + AddRelatedTerms({ + variables: { + input: { + urn: entityDataUrn, + termUrns: selectedUrns, + relationshipType, + }, + }, + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to move: \n ${e.message || ''}`, duration: 3 }); + }) + .finally(() => { + message.loading({ content: 'Adding...', duration: 2 }); + setTimeout(() => { + message.success({ + content: 'Added Related Terms!', + duration: 2, + }); + refetch(); + }, 2000); + }); + onClose(); + } + + const [termSearch, { data: termSearchData }] = useGetSearchResultsLazyQuery(); + const termSearchResults = termSearchData?.search?.searchResults || []; + + const tagSearchOptions = termSearchResults.map((result: SearchResult) => { + const displayName = entityRegistry.getDisplayName(result.entity.type, result.entity); + + return ( + + + + ); + }); + + const handleSearch = (text: string) => { + if (text.length > 0) { + termSearch({ + variables: { + input: { + type: EntityType.GlossaryTerm, + query: text, + start: 0, + count: 20, + }, + }, + }); + } + }; + + // When a Tag or term search result is selected, add the urn to the Urns + const onSelectValue = (urn: string) => { + const newUrns = [...selectedUrns, urn]; + setSelectedUrns(newUrns); + const selectedSearchOption = tagSearchOptions.find((option) => option.props.value === urn); + setSelectedTerms([...selectedTerms, { urn, component: }]); + }; + + // When a Tag or term search result is deselected, remove the urn from the Owners + const onDeselectValue = (urn: string) => { + const newUrns = selectedUrns.filter((u) => u !== urn); + setSelectedUrns(newUrns); + setInputValue(''); + setIsFocusedOnInput(true); + setSelectedTerms(selectedTerms.filter((term) => term.urn !== urn)); + }; + + function selectTermFromBrowser(urn: string, displayName: string) { + setIsFocusedOnInput(false); + const newUrns = [...selectedUrns, urn]; + setSelectedUrns(newUrns); + setSelectedTerms([...selectedTerms, { urn, component: }]); + } + + function clearInput() { + setInputValue(''); + setTimeout(() => setIsFocusedOnInput(true), 0); // call after click outside + } + + function handleBlur() { + setInputValue(''); + } + + const tagRender = (properties) => { + // eslint-disable-next-line react/prop-types + const { closable, onClose: close, value } = properties; + const onPreventMouseDown = (event) => { + event.preventDefault(); + event.stopPropagation(); + }; + const selectedItem = selectedTerms.find((term) => term.urn === value).component; + + return ( + + {selectedItem} + + ); + }; + + const isShowingGlossaryBrowser = !inputValue && isFocusedOnInput; + + return ( + + + + + } + > + setIsFocusedOnInput(false)}> + onSelectValue(asset)} + onDeselect={(asset: any) => onDeselectValue(asset)} + onSearch={(value: string) => { + // eslint-disable-next-line react/prop-types + handleSearch(value.trim()); + // eslint-disable-next-line react/prop-types + setInputValue(value.trim()); + }} + tagRender={tagRender} + value={selectedUrns} + onClear={clearInput} + onFocus={() => setIsFocusedOnInput(true)} + onBlur={handleBlur} + dropdownStyle={isShowingGlossaryBrowser || !inputValue ? { display: 'none' } : {}} + > + {tagSearchOptions} + + + + + + + ); +} + +export default AddRelatedTermsModal; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx index 1129382361b0f4..09abde699b8256 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx @@ -1,7 +1,7 @@ import { Col, Row } from 'antd'; import * as React from 'react'; import styled from 'styled-components'; -import { EmbeddedListSearch } from '../../shared/components/styled/search/EmbeddedListSearch'; +import { EmbeddedListSearchSection } from '../../shared/components/styled/search/EmbeddedListSearchSection'; import { useEntityData } from '../../shared/EntityContext'; @@ -21,7 +21,7 @@ export default function GlossaryRelatedEntity() { return ( - = []; glossaryRelatedTermResult.forEach((item: any) => { glossaryRelatedTermUrns.push(item?.entity?.urn); }); - const glossaryTermInfo: QueryResult>[] = []; - - for (let i = 0; i < glossaryRelatedTermUrns.length; i++) { - glossaryTermInfo.push( - // eslint-disable-next-line react-hooks/rules-of-hooks - useGetGlossaryTermQuery({ - variables: { - urn: glossaryRelatedTermUrns[i], - }, - }), - ); - } + const contentLoading = false; + const relationshipType = + glossaryRelatedTermType === RelatedTermTypes.hasRelatedTerms + ? TermRelationshipType.HasA + : TermRelationshipType.IsA; - const contentLoading = glossaryTermInfo.some((item) => { - return item.loading; - }); return ( <> {contentLoading ? ( @@ -62,28 +53,24 @@ export default function GlossaryRelatedTermsResult({ glossaryRelatedTermType, gl ) : ( - {glossaryRelatedTermType} - + + {glossaryRelatedTermType} + + - { - return ( - - - {entityRegistry.renderPreview( - EntityType.GlossaryTerm, - PreviewType.PREVIEW, - item?.data?.glossaryTerm, - )} - - - - ); - }} - /> + {glossaryRelatedTermUrns.map((urn) => ( + + ))} + {glossaryRelatedTermUrns.length === 0 && ( + + )} )} + {isShowingAddModal && ( + setIsShowingAddModal(false)} relationshipType={relationshipType} /> + )} ); } diff --git a/datahub-web-react/src/app/entity/glossaryTerm/profile/RelatedTerm.tsx b/datahub-web-react/src/app/entity/glossaryTerm/profile/RelatedTerm.tsx new file mode 100644 index 00000000000000..e7f6cc316567a1 --- /dev/null +++ b/datahub-web-react/src/app/entity/glossaryTerm/profile/RelatedTerm.tsx @@ -0,0 +1,77 @@ +import { DeleteOutlined, MoreOutlined } from '@ant-design/icons'; +import { Divider, Dropdown, Menu } from 'antd'; +import React from 'react'; +import styled from 'styled-components/macro'; +import { useGetGlossaryTermQuery } from '../../../../graphql/glossaryTerm.generated'; +import { EntityType, TermRelationshipType } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { PreviewType } from '../../Entity'; +import useRemoveRelatedTerms from './useRemoveRelatedTerms'; + +const ListItem = styled.div` + margin: 0 20px; +`; + +const Profile = styled.div` + display: felx; + marging-bottom: 20px; +`; + +const MenuIcon = styled(MoreOutlined)` + display: flex; + justify-content: center; + align-items: center; + font-size: 20px; + height: 32px; + margin-left: -10px; +`; + +const MenuItem = styled.div` + font-size: 12px; + padding: 0 4px; + color: #262626; +`; + +interface Props { + urn: string; + relationshipType: TermRelationshipType; +} + +function RelatedTerm(props: Props) { + const { urn, relationshipType } = props; + + const entityRegistry = useEntityRegistry(); + const { data, loading } = useGetGlossaryTermQuery({ variables: { urn } }); + let displayName = ''; + if (data) { + displayName = entityRegistry.getDisplayName(EntityType.GlossaryTerm, data.glossaryTerm); + } + const { onRemove } = useRemoveRelatedTerms(urn, relationshipType, displayName); + + if (loading) return null; + + return ( + + + {entityRegistry.renderPreview(EntityType.GlossaryTerm, PreviewType.PREVIEW, data?.glossaryTerm)} + + + +   Remove Term + + + + } + trigger={['click']} + > + + + + + + ); +} + +export default RelatedTerm; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/profile/useRemoveRelatedTerms.tsx b/datahub-web-react/src/app/entity/glossaryTerm/profile/useRemoveRelatedTerms.tsx new file mode 100644 index 00000000000000..0eb46924243cdb --- /dev/null +++ b/datahub-web-react/src/app/entity/glossaryTerm/profile/useRemoveRelatedTerms.tsx @@ -0,0 +1,60 @@ +import { message, Modal } from 'antd'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { useEntityData, useRefetch } from '../../shared/EntityContext'; +import { useRemoveRelatedTermsMutation } from '../../../../graphql/glossaryTerm.generated'; +import { TermRelationshipType } from '../../../../types.generated'; + +function useRemoveRelatedTerms(termUrn: string, relationshipType: TermRelationshipType, displayName: string) { + const { urn, entityType } = useEntityData(); + const entityRegistry = useEntityRegistry(); + const refetch = useRefetch(); + + const [removeRelatedTerms] = useRemoveRelatedTermsMutation(); + + function handleRemoveRelatedTerms() { + removeRelatedTerms({ + variables: { + input: { + urn, + termUrns: [termUrn], + relationshipType, + }, + }, + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to remove: \n ${e.message || ''}`, duration: 3 }); + }) + .finally(() => { + message.loading({ + content: 'Removing...', + duration: 2, + }); + setTimeout(() => { + refetch(); + message.success({ + content: `Removed Glossary Term!`, + duration: 2, + }); + }, 2000); + }); + } + + function onRemove() { + Modal.confirm({ + title: `Remove ${displayName}`, + content: `Are you sure you want to remove this ${entityRegistry.getEntityName(entityType)}?`, + onOk() { + handleRemoveRelatedTerms(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + } + + return { onRemove }; +} + +export default useRemoveRelatedTerms; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/utils.ts b/datahub-web-react/src/app/entity/glossaryTerm/utils.ts new file mode 100644 index 00000000000000..3a2a3d35a8126f --- /dev/null +++ b/datahub-web-react/src/app/entity/glossaryTerm/utils.ts @@ -0,0 +1,8 @@ +import { Entity, EntityType } from '../../../types.generated'; +import EntityRegistry from '../EntityRegistry'; + +export function sortGlossaryTerms(entityRegistry: EntityRegistry, nodeA?: Entity | null, nodeB?: Entity | null) { + const nodeAName = entityRegistry.getDisplayName(EntityType.GlossaryTerm, nodeA) || ''; + const nodeBName = entityRegistry.getDisplayName(EntityType.GlossaryTerm, nodeB) || ''; + return nodeAName.localeCompare(nodeBName); +} diff --git a/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx b/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx index 2d295cda587183..d775b3f98e46b1 100644 --- a/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx +++ b/datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx @@ -1,90 +1,44 @@ import React, { useRef, useState } from 'react'; import { message, Modal, Button, Form, Select, Tag } from 'antd'; import styled from 'styled-components'; -import { Link } from 'react-router-dom'; import { useAddGroupMembersMutation } from '../../../graphql/group.generated'; -import { CorpUser, EntityType, SearchResult } from '../../../types.generated'; -import { CustomAvatar } from '../../shared/avatar'; +import { CorpUser, Entity, EntityType } from '../../../types.generated'; import { useGetSearchResultsLazyQuery } from '../../../graphql/search.generated'; import { useEntityRegistry } from '../../useEntityRegistry'; +import { useGetRecommendations } from '../../shared/recommendation'; +import { OwnerLabel } from '../../shared/OwnerLabel'; type Props = { urn: string; visible: boolean; - onClose: () => void; + onCloseModal: () => void; onSubmit: () => void; }; -const SearchResultContainer = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - padding: 12px; +const SelectInput = styled(Select)` + > .ant-select-selector { + height: 36px; + } `; -const SearchResultContent = styled.div` +const StyleTag = styled(Tag)` + padding: 0px 7px 0px 0px; + margin-right: 3px; display: flex; justify-content: start; align-items: center; `; -const SearchResultDisplayName = styled.div` - margin-left: 12px; -`; - -export const AddGroupMembersModal = ({ urn, visible, onClose, onSubmit }: Props) => { +export const AddGroupMembersModal = ({ urn, visible, onCloseModal, onSubmit }: Props) => { const entityRegistry = useEntityRegistry(); - const [selectedUsers, setSelectedUsers] = useState>([]); - const [userSearch, { data: userSearchData }] = useGetSearchResultsLazyQuery(); + const [selectedMembers, setSelectedMembers] = useState([]); + const [inputValue, setInputValue] = useState(''); const [addGroupMembersMutation] = useAddGroupMembersMutation(); - const searchResults = userSearchData?.search?.searchResults || []; - + const [userSearch, { data: userSearchData }] = useGetSearchResultsLazyQuery(); + const searchResults = userSearchData?.search?.searchResults?.map((searchResult) => searchResult.entity) || []; + const [recommendedData] = useGetRecommendations([EntityType.CorpUser]); const inputEl = useRef(null); - const onAdd = async () => { - if (selectedUsers.length === 0) { - return; - } - addGroupMembersMutation({ - variables: { - groupUrn: urn, - userUrns: selectedUsers.map((user) => user.urn), - }, - }) - .catch((e) => { - message.destroy(); - message.error({ content: `Failed to add group members!: \n ${e.message || ''}`, duration: 3 }); - }) - .finally(() => { - message.success({ - content: `Group members added!`, - duration: 3, - }); - onSubmit(); - setSelectedUsers([]); - }); - onClose(); - }; - - const onSelectMember = (newUserUrn: string) => { - if (inputEl && inputEl.current) { - (inputEl.current as any).blur(); - } - const filteredUsers = searchResults - .filter((result) => result.entity.urn === newUserUrn) - .map((result) => result.entity); - if (filteredUsers.length) { - const newUser = filteredUsers[0] as CorpUser; - const newUsers = [...(selectedUsers || []), newUser]; - setSelectedUsers(newUsers); - } - }; - - const onDeselectMember = (userUrn: string) => { - const newUserActors = selectedUsers.filter((user) => user.urn !== userUrn); - setSelectedUsers(newUserActors); - }; - const handleUserSearch = (text: string) => { if (text.length > 2) { userSearch({ @@ -101,39 +55,95 @@ export const AddGroupMembersModal = ({ urn, visible, onClose, onSubmit }: Props) }; // Renders a search result in the select dropdown. - const renderSearchResult = (result: SearchResult) => { - const avatarUrl = (result.entity as CorpUser).editableProperties?.pictureLink || undefined; - const displayName = entityRegistry.getDisplayName(result.entity.type, result.entity); + const renderSearchResult = (entity: Entity) => { + const avatarUrl = (entity as CorpUser).editableProperties?.pictureLink || undefined; + const displayName = entityRegistry.getDisplayName(entity.type, entity); return ( - - - - -
{displayName}
- - - `/${entityRegistry.getPathName(result.entity.type)}/${result.entity.urn}`} - > - View - {' '} - + + + ); }; + const groupResult = !inputValue || inputValue.length === 0 ? recommendedData : searchResults; + + const groupSearchOptions = groupResult?.map((result) => { + return renderSearchResult(result); + }); + + const onModalClose = () => { + setInputValue(''); + setSelectedMembers([]); + onCloseModal(); + }; + + const onSelectMember = (newMemberUrn: string) => { + if (inputEl && inputEl.current) { + (inputEl.current as any).blur(); + } + const newUsers = [...(selectedMembers || []), newMemberUrn]; + setSelectedMembers(newUsers); + }; + + const onDeselectMember = (memberUrn: string) => { + setInputValue(''); + const newUserActors = selectedMembers.filter((user) => user !== memberUrn); + setSelectedMembers(newUserActors); + }; + + const tagRender = (props) => { + // eslint-disable-next-line react/prop-types + const { label, closable, onClose } = props; + const onPreventMouseDown = (event) => { + event.preventDefault(); + event.stopPropagation(); + }; + return ( + + {label} + + ); + }; + + const onAdd = async () => { + const selectedMemberUrns = selectedMembers.map((selectedMember) => selectedMember.value); + if (selectedMembers.length === 0) { + return; + } + try { + await addGroupMembersMutation({ + variables: { + groupUrn: urn, + userUrns: selectedMemberUrns, + }, + }); + message.success({ content: 'Group members added!', duration: 3 }); + } catch (e: unknown) { + message.destroy(); + if (e instanceof Error) { + message.error({ content: `Failed to group members: \n ${e.message || ''}`, duration: 3 }); + } + } finally { + onSubmit(); + onModalClose(); + } + }; + + function handleBlur() { + setInputValue(''); + } + return ( - - @@ -141,23 +151,30 @@ export const AddGroupMembersModal = ({ urn, visible, onClose, onSubmit }: Props) >
- + {groupSearchOptions} +
diff --git a/datahub-web-react/src/app/entity/group/Group.tsx b/datahub-web-react/src/app/entity/group/Group.tsx index be5d69fe7dc5f8..89c5823428c4ec 100644 --- a/datahub-web-react/src/app/entity/group/Group.tsx +++ b/datahub-web-react/src/app/entity/group/Group.tsx @@ -62,10 +62,14 @@ export class GroupEntity implements Entity { }; displayName = (data: CorpGroup) => { - return data.properties?.displayName || data.info?.displayName || data.name; + return data.properties?.displayName || data.info?.displayName || data.name || data.urn; }; getGenericEntityProperties = (group: CorpGroup) => { return getDataForEntityType({ data: group, entityType: this.type, getOverrideProperties: (data) => data }); }; + + supportedCapabilities = () => { + return new Set([]); + }; } diff --git a/datahub-web-react/src/app/entity/group/GroupAssets.tsx b/datahub-web-react/src/app/entity/group/GroupAssets.tsx index d015cc7e33e787..3dd1e9f97e4412 100644 --- a/datahub-web-react/src/app/entity/group/GroupAssets.tsx +++ b/datahub-web-react/src/app/entity/group/GroupAssets.tsx @@ -1,6 +1,6 @@ import React from 'react'; import styled from 'styled-components'; -import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch'; +import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection'; const GroupAssetsWrapper = styled.div` height: calc(100vh - 114px); @@ -13,7 +13,7 @@ type Props = { export const GroupAssets = ({ urn }: Props) => { return ( - {name} + + {isExternalGroup && ( + + + + )} + diff --git a/datahub-web-react/src/app/entity/group/GroupMembers.tsx b/datahub-web-react/src/app/entity/group/GroupMembers.tsx index 63a06f64a90e64..cd55e48a52b9c7 100644 --- a/datahub-web-react/src/app/entity/group/GroupMembers.tsx +++ b/datahub-web-react/src/app/entity/group/GroupMembers.tsx @@ -1,9 +1,9 @@ import React, { useState } from 'react'; import { MoreOutlined, UserAddOutlined, UserDeleteOutlined } from '@ant-design/icons'; -import { Col, Dropdown, Menu, message, Modal, Pagination, Row, Empty } from 'antd'; +import { Col, Dropdown, Menu, message, Modal, Pagination, Row, Empty, Button, Typography } from 'antd'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; -import { useGetGroupMembersQuery, useRemoveGroupMembersMutation } from '../../../graphql/group.generated'; +import { useGetAllGroupMembersQuery, useRemoveGroupMembersMutation } from '../../../graphql/group.generated'; import { CorpUser, EntityType } from '../../../types.generated'; import { CustomAvatar } from '../../shared/avatar'; import { useEntityRegistry } from '../../useEntityRegistry'; @@ -18,14 +18,8 @@ const AVATAR_STYLE = { margin: '5px 5px 5px 0' }; /** * Styled Components */ -const AddMember = styled(Col)` - font-family: Manrope; - font-style: normal; - font-weight: 500; - font-size: 12px; - line-height: 20px; - color: #262626; - padding: 13px 30px; +const AddMember = styled(Button)` + padding: 13px 13px 30px 30px; cursor: pointer; &&& .anticon.anticon-user-add { @@ -33,6 +27,14 @@ const AddMember = styled(Col)` } `; +const AddMemberText = styled(Typography.Text)` + font-family: Manrope; + font-style: normal; + font-weight: 500; + font-size: 12px; + line-height: 20px; +`; + const MemberNameSection = styled.div` font-size: 20px; line-height: 28px; @@ -77,17 +79,18 @@ const NoGroupMembers = styled(Empty)` type Props = { urn: string; pageSize: number; + isExternalGroup: boolean; onChangeMembers?: () => void; }; -export default function GroupMembers({ urn, pageSize, onChangeMembers }: Props) { +export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeMembers }: Props) { const entityRegistry = useEntityRegistry(); const [page, setPage] = useState(1); /* eslint-disable @typescript-eslint/no-unused-vars */ const [isEditingMembers, setIsEditingMembers] = useState(false); const start = (page - 1) * pageSize; - const { data: membersData, refetch } = useGetGroupMembersQuery({ + const { data: membersData, refetch } = useGetAllGroupMembersQuery({ variables: { urn, start, count: pageSize }, }); const [removeGroupMembersMutation] = useRemoveGroupMembersMutation(); @@ -129,7 +132,7 @@ export default function GroupMembers({ urn, pageSize, onChangeMembers }: Props) setTimeout(function () { // Reload the page. refetch(); - }, 2000); + }, 3000); }; const onRemoveMember = (memberUrn: string) => { @@ -165,7 +168,7 @@ export default function GroupMembers({ urn, pageSize, onChangeMembers }: Props) Make owner - + Remove from Group @@ -177,9 +180,9 @@ export default function GroupMembers({ urn, pageSize, onChangeMembers }: Props) return ( <> - + - Add Member + Add Member @@ -224,12 +227,14 @@ export default function GroupMembers({ urn, pageSize, onChangeMembers }: Props) showSizeChanger={false} /> - setIsEditingMembers(false)} - /> + {isEditingMembers && ( + setIsEditingMembers(false)} + /> + )} ); } diff --git a/datahub-web-react/src/app/entity/group/GroupOwnerSideBarSection.tsx b/datahub-web-react/src/app/entity/group/GroupOwnerSideBarSection.tsx index 1708d6f5aa62b5..90f5c43e0432c3 100644 --- a/datahub-web-react/src/app/entity/group/GroupOwnerSideBarSection.tsx +++ b/datahub-web-react/src/app/entity/group/GroupOwnerSideBarSection.tsx @@ -4,7 +4,7 @@ import React, { useState } from 'react'; import styled from 'styled-components'; import { EntityType, Ownership } from '../../../types.generated'; import { ExpandedOwner } from '../shared/components/styled/ExpandedOwner'; -import { AddOwnersModal } from '../shared/containers/profile/sidebar/Ownership/AddOwnersModal'; +import { EditOwnersModal } from '../shared/containers/profile/sidebar/Ownership/EditOwnersModal'; import { DisplayCount, GroupSectionTitle, GroupSectionHeader } from '../shared/SidebarStyledComponents'; const TITLE = 'Owners'; @@ -50,16 +50,17 @@ export default function GroupOwnerSideBarSection({ urn, ownership, refetch }: Pr )} - { - setShowAddModal(false); - }} - /> + {showAddModal && ( + { + setShowAddModal(false); + }} + /> + )} ); } diff --git a/datahub-web-react/src/app/entity/group/GroupProfile.tsx b/datahub-web-react/src/app/entity/group/GroupProfile.tsx index f401b7bb15a3a7..616236ead2ac59 100644 --- a/datahub-web-react/src/app/entity/group/GroupProfile.tsx +++ b/datahub-web-react/src/app/entity/group/GroupProfile.tsx @@ -3,7 +3,7 @@ import { Alert, Col, Row } from 'antd'; import styled from 'styled-components'; import { useGetGroupQuery } from '../../../graphql/group.generated'; import useUserParams from '../../shared/entitySearch/routingUtils/useUserParams'; -import { EntityRelationshipsResult, Ownership } from '../../../types.generated'; +import { OriginType, EntityRelationshipsResult, Ownership } from '../../../types.generated'; import { Message } from '../../shared/Message'; import GroupMembers from './GroupMembers'; import { decodeUrn } from '../shared/utils'; @@ -53,6 +53,8 @@ export default function GroupProfile() { } const groupMemberRelationships = data?.corpGroup?.relationships as EntityRelationshipsResult; + const isExternalGroup: boolean = data?.corpGroup?.origin?.type === OriginType.External; + const externalGroupType: string = data?.corpGroup?.origin?.externalType || 'outside DataHub'; const getTabs = () => { return [ @@ -71,6 +73,7 @@ export default function GroupProfile() { { setTimeout(() => refetch(), 2000); }} @@ -105,6 +108,8 @@ export default function GroupProfile() { data?.corpGroup?.editableProperties?.description || data?.corpGroup?.properties?.description || undefined, groupMemberRelationships: groupMemberRelationships as EntityRelationshipsResult, groupOwnerShip: data?.corpGroup?.ownership as Ownership, + isExternalGroup, + externalGroupType, urn, }; diff --git a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx index 73aba5e4b0824f..98781da4f9f5d3 100644 --- a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { DotChartOutlined } from '@ant-design/icons'; import { MlFeature, EntityType, SearchResult, OwnershipType } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; import { GenericEntityProperties } from '../shared/types'; @@ -150,7 +150,7 @@ export class MLFeatureEntity implements Entity { }; displayName = (data: MlFeature) => { - return data.name; + return data.name || data.urn; }; getGenericEntityProperties = (mlFeature: MlFeature) => { @@ -172,4 +172,15 @@ export class MLFeatureEntity implements Entity { platform: entity?.['featureTables']?.relationships?.[0]?.entity?.platform?.name, }; }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/mlFeature/preview/Preview.tsx b/datahub-web-react/src/app/entity/mlFeature/preview/Preview.tsx index e21bf67ce20b37..163831d6c65641 100644 --- a/datahub-web-react/src/app/entity/mlFeature/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/mlFeature/preview/Preview.tsx @@ -3,6 +3,7 @@ import { DataPlatform, EntityType, Owner } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { capitalizeFirstLetterOnly } from '../../../shared/textUtil'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import { IconStyleType } from '../../Entity'; export const Preview = ({ urn, @@ -30,7 +31,8 @@ export const Preview = ({ description={description || ''} platform={capitalizeFirstLetterOnly(platform?.properties?.displayName) || featureNamespace} logoUrl={platform?.properties?.logoUrl || ''} - type="MLFeature" + type="ML Feature" + typeIcon={entityRegistry.getIcon(EntityType.Mlfeature, 14, IconStyleType.ACCENT)} owners={owners} /> ); diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx index 9b956e0a4a236d..3aa4fbc86afc4c 100644 --- a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { DotChartOutlined } from '@ant-design/icons'; import { MlFeatureTable, EntityType, SearchResult, OwnershipType } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { GenericEntityProperties } from '../shared/types'; import { useGetMlFeatureTableQuery } from '../../../graphql/mlFeatureTable.generated'; @@ -149,7 +149,7 @@ export class MLFeatureTableEntity implements Entity { }; displayName = (data: MlFeatureTable) => { - return data.name; + return data.name || data.urn; }; getGenericEntityProperties = (mlFeatureTable: MlFeatureTable) => { @@ -159,4 +159,15 @@ export class MLFeatureTableEntity implements Entity { getOverrideProperties: (data) => data, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/preview/Preview.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/preview/Preview.tsx index 6a1587f56e9051..84e7a2ac475a23 100644 --- a/datahub-web-react/src/app/entity/mlFeatureTable/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/mlFeatureTable/preview/Preview.tsx @@ -28,6 +28,7 @@ export const Preview = ({ name={name} description={description || ''} type={entityRegistry.getEntityName(EntityType.MlfeatureTable)} + typeIcon={entityRegistry.getIcon(EntityType.MlfeatureTable, 14, IconStyleType.ACCENT)} owners={owners} logoUrl={logoUrl || undefined} platform={platformName || ''} diff --git a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx index be63b851c8ac79..329ecde7af9f14 100644 --- a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { CodeSandboxOutlined } from '@ant-design/icons'; import { MlModel, EntityType, SearchResult, OwnershipType } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; import { useGetMlModelQuery } from '../../../graphql/mlModel.generated'; @@ -135,10 +135,21 @@ export class MLModelEntity implements Entity { }; displayName = (data: MlModel) => { - return data.name; + return data.name || data.urn; }; getGenericEntityProperties = (mlModel: MlModel) => { return getDataForEntityType({ data: mlModel, entityType: this.type, getOverrideProperties: (data) => data }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx b/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx index 70151cdc4d112b..8f5d3449d827a4 100644 --- a/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/mlModel/preview/Preview.tsx @@ -1,15 +1,10 @@ import React from 'react'; -import styled from 'styled-components'; import { EntityType, MlModel } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { capitalizeFirstLetter } from '../../../shared/textUtil'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { IconStyleType } from '../../Entity'; -const LogoContainer = styled.div` - padding-right: 8px; -`; - export const Preview = ({ model }: { model: MlModel }): JSX.Element => { const entityRegistry = useEntityRegistry(); const capitalPlatformName = capitalizeFirstLetter(model?.platform?.name || ''); @@ -21,9 +16,7 @@ export const Preview = ({ model }: { model: MlModel }): JSX.Element => { description={model.description || ''} platformInstanceId={model.dataPlatformInstance?.instanceId} type={entityRegistry.getEntityName(EntityType.Mlmodel)} - logoComponent={ - {entityRegistry.getIcon(EntityType.Mlmodel, 20, IconStyleType.HIGHLIGHT)} - } + typeIcon={entityRegistry.getIcon(EntityType.Mlmodel, 14, IconStyleType.ACCENT)} platform={capitalPlatformName} qualifier={model.origin} tags={model.globalTags || undefined} diff --git a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx index 3180936ec9e431..a42809aa078c79 100644 --- a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { CodeSandboxOutlined } from '@ant-design/icons'; import { MlModelGroup, EntityType, SearchResult, OwnershipType } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { GenericEntityProperties } from '../shared/types'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; @@ -120,7 +120,7 @@ export class MLModelGroupEntity implements Entity { }; displayName = (data: MlModelGroup) => { - return data.name; + return data.name || data.urn; }; getGenericEntityProperties = (mlModelGroup: MlModelGroup) => { @@ -130,4 +130,15 @@ export class MLModelGroupEntity implements Entity { getOverrideProperties: (data) => data, }); }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx index cf2d3ddcbce263..647e1c051e5a92 100644 --- a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx +++ b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { DotChartOutlined } from '@ant-design/icons'; import { MlPrimaryKey, EntityType, SearchResult, OwnershipType } from '../../../types.generated'; import { Preview } from './preview/Preview'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { GenericEntityProperties } from '../shared/types'; import { GetMlPrimaryKeyQuery, useGetMlPrimaryKeyQuery } from '../../../graphql/mlPrimaryKey.generated'; @@ -148,7 +148,7 @@ export class MLPrimaryKeyEntity implements Entity { }; displayName = (data: MlPrimaryKey) => { - return data.name; + return data.name || data.urn; }; getGenericEntityProperties = (mlPrimaryKey: MlPrimaryKey) => { @@ -170,4 +170,15 @@ export class MLPrimaryKeyEntity implements Entity { platform: entity?.['featureTables']?.relationships?.[0]?.entity?.platform?.name, }; }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.TAGS, + EntityCapabilityType.DOMAINS, + EntityCapabilityType.DEPRECATION, + EntityCapabilityType.SOFT_DELETE, + ]); + }; } diff --git a/datahub-web-react/src/app/entity/mlPrimaryKey/preview/Preview.tsx b/datahub-web-react/src/app/entity/mlPrimaryKey/preview/Preview.tsx index 9e3f47a9c2c02a..4b2c927abb7a1f 100644 --- a/datahub-web-react/src/app/entity/mlPrimaryKey/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/mlPrimaryKey/preview/Preview.tsx @@ -3,6 +3,7 @@ import { DataPlatform, EntityType, Owner } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { capitalizeFirstLetterOnly } from '../../../shared/textUtil'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import { IconStyleType } from '../../Entity'; export const Preview = ({ urn, @@ -29,7 +30,8 @@ export const Preview = ({ description={description || ''} platform={capitalizeFirstLetterOnly(platform?.properties?.displayName) || featureNamespace} logoUrl={platform?.properties?.logoUrl || ''} - type="MLPrimaryKey" + type="ML Primary Key" + typeIcon={entityRegistry.getIcon(EntityType.MlprimaryKey, 14, IconStyleType.ACCENT)} owners={owners} platformInstanceId={platformInstanceId} /> diff --git a/datahub-web-react/src/app/entity/shared/EntityContext.tsx b/datahub-web-react/src/app/entity/shared/EntityContext.tsx index 4001646b2f2dcf..c6b31c1b7781ba 100644 --- a/datahub-web-react/src/app/entity/shared/EntityContext.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityContext.tsx @@ -1,5 +1,6 @@ import React, { useContext } from 'react'; import { EntityType } from '../../../types.generated'; +import { useIsSeparateSiblingsMode } from './siblingUtils'; import { EntityContextType, UpdateEntityType } from './types'; const EntityContext = React.createContext({ @@ -11,6 +12,7 @@ const EntityContext = React.createContext({ routeToTab: () => {}, refetch: () => Promise.resolve({}), lineage: undefined, + dataNotCombinedWithSiblings: null, }); export default EntityContext; @@ -20,6 +22,11 @@ export const useBaseEntity = (): T => { return baseEntity as T; }; +export const useDataNotCombinedWithSiblings = (): T => { + const { dataNotCombinedWithSiblings } = useContext(EntityContext); + return dataNotCombinedWithSiblings as T; +}; + export const useEntityUpdate = (): UpdateEntityType | null | undefined => { const { updateEntity } = useContext(EntityContext); return updateEntity; @@ -44,3 +51,12 @@ export const useLineageData = () => { const { lineage } = useContext(EntityContext); return lineage; }; + +export const useMutationUrn = () => { + const { urn, entityData } = useContext(EntityContext); + const isHideSiblingMode = useIsSeparateSiblingsMode(); + if (!entityData?.siblings || entityData?.siblings?.isPrimary || isHideSiblingMode) { + return urn; + } + return entityData?.siblings?.siblings?.[0]?.urn || urn; +}; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx index f38b9964b61e17..f136ea76d7dc77 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/CreateGlossaryEntityModal.tsx @@ -49,11 +49,7 @@ function CreateGlossaryEntityModal(props: Props) { }, }, }) - .catch((e) => { - message.destroy(); - message.error({ content: `Failed to create: \n ${e.message || ''}`, duration: 3 }); - }) - .finally(() => { + .then(() => { message.loading({ content: 'Updating...', duration: 2 }); setTimeout(() => { message.success({ @@ -65,6 +61,10 @@ function CreateGlossaryEntityModal(props: Props) { refetchData(); } }, 2000); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to create: \n ${e.message || ''}`, duration: 3 }); }); onClose(); } @@ -104,7 +104,7 @@ function CreateGlossaryEntityModal(props: Props) { message: `Enter a ${entityRegistry.getEntityName(entityType)} name.`, }, { whitespace: true }, - { min: 1, max: 50 }, + { min: 1, max: 100 }, ]} hasFeedback > diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index 90781950187b5a..bf3e46b6582e6c 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -13,15 +13,14 @@ import { import { Redirect } from 'react-router'; import { EntityType, PlatformPrivileges } from '../../../../types.generated'; import CreateGlossaryEntityModal from './CreateGlossaryEntityModal'; -import { AddDeprecationDetailsModal } from './AddDeprecationDetailsModal'; -import { useEntityData, useRefetch } from '../EntityContext'; +import { UpdateDeprecationModal } from './UpdateDeprecationModal'; import { useUpdateDeprecationMutation } from '../../../../graphql/mutations.generated'; import MoveGlossaryEntityModal from './MoveGlossaryEntityModal'; -import useDeleteGlossaryEntity from './useDeleteGlossaryEntity'; -import { PageRoutes } from '../../../../conf/Global'; import { ANTD_GRAY } from '../constants'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { useGetAuthenticatedUser } from '../../../useGetAuthenticatedUser'; +import useDeleteEntity from './useDeleteEntity'; +import { getEntityProfileDeleteRedirectPath } from '../../../shared/deleteUtils'; export enum EntityMenuItems { COPY_URL, @@ -32,11 +31,11 @@ export enum EntityMenuItems { MOVE, } -const MenuIcon = styled(MoreOutlined)` +const MenuIcon = styled(MoreOutlined)<{ fontSize?: number }>` display: flex; justify-content: center; align-items: center; - font-size: 25px; + font-size: ${(props) => props.fontSize || '24'}px; height: 32px; margin-left: 5px; `; @@ -59,22 +58,38 @@ const StyledMenuItem = styled(Menu.Item)<{ disabled: boolean }>` `; interface Props { + urn: string; + entityType: EntityType; + entityData?: any; menuItems: Set; platformPrivileges?: PlatformPrivileges; + size?: number; + refetchForEntity?: () => void; refetchForTerms?: () => void; refetchForNodes?: () => void; refreshBrowser?: () => void; + onDeleteEntity?: () => void; } function EntityDropdown(props: Props) { - const { menuItems, platformPrivileges, refetchForTerms, refetchForNodes, refreshBrowser } = props; + const { + urn, + entityData, + entityType, + menuItems, + platformPrivileges, + refetchForEntity, + refetchForTerms, + refetchForNodes, + refreshBrowser, + onDeleteEntity: onDelete, + size, + } = props; const entityRegistry = useEntityRegistry(); - const { urn, entityData, entityType } = useEntityData(); - const refetch = useRefetch(); const me = useGetAuthenticatedUser(!!platformPrivileges); const [updateDeprecation] = useUpdateDeprecationMutation(); - const { onDeleteEntity, hasBeenDeleted } = useDeleteGlossaryEntity(); + const { onDeleteEntity, hasBeenDeleted } = useDeleteEntity(urn, entityType, entityData, onDelete); const [isCreateTermModalVisible, setIsCreateTermModalVisible] = useState(false); const [isCreateNodeModalVisible, setIsCreateNodeModalVisible] = useState(false); @@ -102,15 +117,20 @@ function EntityDropdown(props: Props) { message.error({ content: `Failed to update Deprecation: \n ${e.message || ''}`, duration: 2 }); } } - refetch?.(); + refetchForEntity?.(); }; const canManageGlossaries = platformPrivileges - ? platformPrivileges.manageDomains - : me?.platformPrivileges.manageDomains; + ? platformPrivileges.manageGlossaries + : me?.platformPrivileges.manageGlossaries; const pageUrl = window.location.href; const isDeleteDisabled = !!entityData?.children?.count; + /** + * A default path to redirect to if the entity is deleted. + */ + const deleteRedirectPath = getEntityProfileDeleteRedirectPath(entityType); + return ( <> )} {menuItems.has(EntityMenuItems.ADD_TERM) && ( - - setIsCreateTermModalVisible(true)}> + setIsCreateTermModalVisible(true)} + > +  Add Term )} {menuItems.has(EntityMenuItems.ADD_TERM_GROUP) && ( - - setIsCreateNodeModalVisible(true)}> + setIsCreateNodeModalVisible(true)} + > +  Add Term Group )} {menuItems.has(EntityMenuItems.MOVE) && ( - - setIsMoveModalVisible(true)}> + setIsMoveModalVisible(true)} + > +  Move )} {menuItems.has(EntityMenuItems.DELETE) && ( - + - +  Delete @@ -180,7 +216,7 @@ function EntityDropdown(props: Props) { } trigger={['click']} > - + {isCreateTermModalVisible && ( )} {isDeprecationModalVisible && ( - setIsDeprecationModalVisible(false)} - refetch={refetch} + refetch={refetchForEntity} /> )} {isMoveModalVisible && ( setIsMoveModalVisible(false)} refetchData={refreshBrowser} /> )} - {hasBeenDeleted && } + {hasBeenDeleted && !onDelete && deleteRedirectPath && } ); } diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx index e2228c8be5f25a..631530652944bd 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx @@ -43,11 +43,7 @@ function MoveGlossaryEntityModal(props: Props) { }, }, }) - .catch((e) => { - message.destroy(); - message.error({ content: `Failed to move: \n ${e.message || ''}`, duration: 3 }); - }) - .finally(() => { + .then(() => { message.loading({ content: 'Updating...', duration: 2 }); setTimeout(() => { message.success({ @@ -59,6 +55,10 @@ function MoveGlossaryEntityModal(props: Props) { refetchData(); } }, 2000); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to move: \n ${e.message || ''}`, duration: 3 }); }); onClose(); } diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx index e2b24ae9e78a4d..86c2b84a67c3d1 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx @@ -29,7 +29,7 @@ function NodeParentSelect(props: Props) { const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); const [searchQuery, setSearchQuery] = useState(''); const entityRegistry = useEntityRegistry(); - const { entityData, urn: entityDataUrn } = useEntityData(); + const { entityData, urn: entityDataUrn, entityType } = useEntityData(); const [nodeSearch, { data: nodeData }] = useGetSearchResultsLazyQuery(); let nodeSearchResults = nodeData?.search?.searchResults || []; @@ -82,6 +82,7 @@ function NodeParentSelect(props: Props) { } const isShowingGlossaryBrowser = !searchQuery && isFocusedOnInput; + const shouldHideSelf = isMoving && entityType === EntityType.GlossaryNode; return ( setIsFocusedOnInput(false)}> @@ -103,7 +104,12 @@ function NodeParentSelect(props: Props) { ))} - + ); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/AddDeprecationDetailsModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx similarity index 82% rename from datahub-web-react/src/app/entity/shared/EntityDropdown/AddDeprecationDetailsModal.tsx rename to datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx index 6fba13d1648500..db91ec888f2921 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/AddDeprecationDetailsModal.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/UpdateDeprecationModal.tsx @@ -1,16 +1,15 @@ import React from 'react'; import { Button, DatePicker, Form, Input, message, Modal } from 'antd'; -import { useUpdateDeprecationMutation } from '../../../../graphql/mutations.generated'; +import { useBatchUpdateDeprecationMutation } from '../../../../graphql/mutations.generated'; type Props = { - urn: string; - visible: boolean; + urns: string[]; onClose: () => void; - refetch?: () => Promise; + refetch?: () => void; }; -export const AddDeprecationDetailsModal = ({ urn, visible, onClose, refetch }: Props) => { - const [updateDeprecation] = useUpdateDeprecationMutation(); +export const UpdateDeprecationModal = ({ urns, onClose, refetch }: Props) => { + const [batchUpdateDeprecation] = useBatchUpdateDeprecationMutation(); const [form] = Form.useForm(); const handleClose = () => { @@ -21,10 +20,10 @@ export const AddDeprecationDetailsModal = ({ urn, visible, onClose, refetch }: P const handleOk = async (formData: any) => { message.loading({ content: 'Updating...' }); try { - await updateDeprecation({ + await batchUpdateDeprecation({ variables: { input: { - urn, + resources: [...urns.map((urn) => ({ resourceUrn: urn }))], deprecated: true, note: formData.note, decommissionTime: formData.decommissionTime && formData.decommissionTime.unix(), @@ -46,7 +45,7 @@ export const AddDeprecationDetailsModal = ({ urn, visible, onClose, refetch }: P return ( void) { + const [hasBeenDeleted, setHasBeenDeleted] = useState(false); + const entityRegistry = useEntityRegistry(); + + const maybeDeleteEntity = getDeleteEntityMutation(type)(); + const deleteEntity = (maybeDeleteEntity && maybeDeleteEntity[0]) || undefined; + + function handleDeleteEntity() { + deleteEntity?.({ + variables: { + urn, + }, + }) + .then(() => { + message.loading({ + content: 'Deleting...', + duration: 2, + }); + setTimeout(() => { + setHasBeenDeleted(true); + onDelete?.(); + message.success({ + content: `Deleted ${entityRegistry.getEntityName(type)}!`, + duration: 2, + }); + }, 2000); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to delete: \n ${e.message || ''}`, duration: 3 }); + }); + } + + function onDeleteEntity() { + Modal.confirm({ + title: `Delete ${ + (entityData && entityRegistry.getDisplayName(type, entityData)) || entityRegistry.getEntityName(type) + }`, + content: `Are you sure you want to remove this ${entityRegistry.getEntityName(type)}?`, + onOk() { + handleDeleteEntity(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + } + + return { onDeleteEntity, hasBeenDeleted }; +} + +export default useDeleteEntity; diff --git a/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts new file mode 100644 index 00000000000000..32a678919537ff --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts @@ -0,0 +1,476 @@ +import { dataset3WithLineage, dataset4WithLineage } from '../../../../Mocks'; +import { EntityType } from '../../../../types.generated'; +import { combineEntityDataWithSiblings, combineSiblingsInSearchResults } from '../siblingUtils'; + +const usageStats = { + buckets: [ + { + bucket: 1650412800000, + duration: 'DAY', + resource: 'urn:li:dataset:4', + metrics: { + uniqueUserCount: 1, + totalSqlQueries: 37, + topSqlQueries: ['some sql query'], + __typename: 'UsageAggregationMetrics', + }, + __typename: 'UsageAggregation', + }, + ], + aggregations: { + uniqueUserCount: 2, + totalSqlQueries: 39, + users: [ + { + user: { + urn: 'urn:li:corpuser:user', + username: 'user', + __typename: 'CorpUser', + }, + count: 2, + userEmail: 'user@datahubproject.io', + __typename: 'UserUsageCounts', + }, + ], + fields: [ + { + fieldName: 'field', + count: 7, + __typename: 'FieldUsageCounts', + }, + ], + __typename: 'UsageQueryResultAggregations', + }, + __typename: 'UsageQueryResult', +}; + +const datasetPrimary = { + ...dataset3WithLineage, + properties: { + ...dataset3WithLineage.properties, + description: 'primary description', + }, + editableProperties: { + description: '', + }, + globalTags: { + tags: [ + { + tag: { + type: EntityType.Tag, + urn: 'urn:li:tag:primary-tag', + name: 'primary-tag', + description: 'primary tag', + properties: { + name: 'primary-tag', + description: 'primary tag', + colorHex: 'primary tag color', + }, + }, + }, + ], + }, +}; +const datasetUnprimary = { + ...dataset4WithLineage, + usageStats, + properties: { + ...dataset4WithLineage.properties, + description: 'unprimary description', + }, + editableProperties: { + description: 'secondary description', + }, + globalTags: { + tags: [ + { + tag: { + type: EntityType.Tag, + urn: 'urn:li:tag:unprimary-tag', + name: 'unprimary-tag', + description: 'unprimary tag', + properties: { + name: 'unprimary-tag', + description: 'unprimary tag', + colorHex: 'unprimary tag color', + }, + }, + }, + ], + }, +}; + +const datasetPrimaryWithSiblings = { + ...datasetPrimary, + siblings: { + isPrimary: true, + siblings: [datasetUnprimary], + }, +}; + +const searchResultWithSiblings = [ + { + entity: { + urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', + type: 'DATASET', + name: 'cypress_project.jaffle_shop.raw_orders', + origin: 'PROD', + uri: null, + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + dataPlatformInstance: null, + editableProperties: null, + platformNativeType: null, + properties: { + name: 'raw_orders', + description: null, + qualifiedName: null, + customProperties: [], + __typename: 'DatasetProperties', + }, + ownership: null, + globalTags: null, + glossaryTerms: null, + subTypes: { + typeNames: ['table'], + __typename: 'SubTypes', + }, + domain: null, + container: { + urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + properties: { + name: 'jaffle_shop', + __typename: 'ContainerProperties', + }, + subTypes: { + typeNames: ['Dataset'], + __typename: 'SubTypes', + }, + deprecation: null, + __typename: 'Container', + }, + parentContainers: { + count: 2, + containers: [ + { + urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + properties: { + name: 'jaffle_shop', + __typename: 'ContainerProperties', + }, + subTypes: { + typeNames: ['Dataset'], + __typename: 'SubTypes', + }, + deprecation: null, + __typename: 'Container', + }, + { + urn: 'urn:li:container:b5e95fce839e7d78151ed7e0a7420d84', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + properties: { + name: 'cypress_project', + __typename: 'ContainerProperties', + }, + subTypes: { + typeNames: ['Project'], + __typename: 'SubTypes', + }, + deprecation: null, + __typename: 'Container', + }, + ], + __typename: 'ParentContainersResult', + }, + deprecation: null, + siblings: { + isPrimary: false, + siblings: [ + { + urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)', + type: 'DATASET', + platform: { + urn: 'urn:li:dataPlatform:dbt', + type: 'DATA_PLATFORM', + name: 'dbt', + properties: { + type: 'OTHERS', + displayName: 'dbt', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/dbtlogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + name: 'cypress_project.jaffle_shop.raw_orders', + properties: { + name: 'raw_orders', + description: '', + qualifiedName: null, + __typename: 'DatasetProperties', + }, + __typename: 'Dataset', + }, + ], + __typename: 'SiblingProperties', + }, + __typename: 'Dataset', + }, + matchedFields: [ + { + name: 'name', + value: 'raw_orders', + __typename: 'MatchedField', + }, + { + name: 'id', + value: 'cypress_project.jaffle_shop.raw_orders', + __typename: 'MatchedField', + }, + ], + insights: [], + __typename: 'SearchResult', + }, + { + entity: { + urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)', + type: 'DATASET', + name: 'cypress_project.jaffle_shop.raw_orders', + origin: 'PROD', + uri: null, + platform: { + urn: 'urn:li:dataPlatform:dbt', + type: 'DATA_PLATFORM', + name: 'dbt', + properties: { + type: 'OTHERS', + displayName: 'dbt', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/dbtlogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + dataPlatformInstance: null, + editableProperties: null, + platformNativeType: null, + properties: { + name: 'raw_orders', + description: '', + qualifiedName: null, + customProperties: [ + { + key: 'catalog_version', + value: '1.0.4', + __typename: 'StringMapEntry', + }, + { + key: 'node_type', + value: 'seed', + __typename: 'StringMapEntry', + }, + { + key: 'materialization', + value: 'seed', + __typename: 'StringMapEntry', + }, + { + key: 'dbt_file_path', + value: 'data/raw_orders.csv', + __typename: 'StringMapEntry', + }, + { + key: 'catalog_schema', + value: 'https://schemas.getdbt.com/dbt/catalog/v1.json', + __typename: 'StringMapEntry', + }, + { + key: 'catalog_type', + value: 'table', + __typename: 'StringMapEntry', + }, + { + key: 'manifest_version', + value: '1.0.4', + __typename: 'StringMapEntry', + }, + { + key: 'manifest_schema', + value: 'https://schemas.getdbt.com/dbt/manifest/v4.json', + __typename: 'StringMapEntry', + }, + ], + __typename: 'DatasetProperties', + }, + ownership: null, + globalTags: null, + glossaryTerms: null, + subTypes: { + typeNames: ['seed'], + __typename: 'SubTypes', + }, + domain: null, + container: null, + parentContainers: { + count: 0, + containers: [], + __typename: 'ParentContainersResult', + }, + deprecation: null, + siblings: { + isPrimary: true, + siblings: [ + { + urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', + type: 'DATASET', + platform: { + urn: 'urn:li:dataPlatform:bigquery', + type: 'DATA_PLATFORM', + name: 'bigquery', + properties: { + type: 'RELATIONAL_DB', + displayName: 'BigQuery', + datasetNameDelimiter: '.', + logoUrl: '/assets/platforms/bigquerylogo.png', + __typename: 'DataPlatformProperties', + }, + displayName: null, + info: null, + __typename: 'DataPlatform', + }, + name: 'cypress_project.jaffle_shop.raw_orders', + properties: { + name: 'raw_orders', + description: null, + qualifiedName: null, + __typename: 'DatasetProperties', + }, + __typename: 'Dataset', + }, + ], + __typename: 'SiblingProperties', + }, + __typename: 'Dataset', + }, + matchedFields: [ + { + name: 'name', + value: 'raw_orders', + __typename: 'MatchedField', + }, + { + name: 'id', + value: 'cypress_project.jaffle_shop.raw_orders', + __typename: 'MatchedField', + }, + ], + insights: [], + __typename: 'SearchResult', + }, +]; + +// const datasetUnprimaryWithSiblings = { +// ...datasetUnprimary, +// siblings: { +// isPrimary: true, +// siblings: [datasetPrimary], +// }, +// }; + +describe('siblingUtils', () => { + describe('combineEntityDataWithSiblings', () => { + it('combines my metadata with my siblings', () => { + const baseEntity = { dataset: datasetPrimaryWithSiblings }; + expect(baseEntity.dataset.usageStats).toBeNull(); + const combinedData = combineEntityDataWithSiblings(baseEntity); + // will merge properties only one entity has + expect(combinedData.dataset.usageStats).toEqual(usageStats); + + // will merge arrays + expect(combinedData.dataset.globalTags.tags).toHaveLength(2); + expect(combinedData.dataset.globalTags.tags[0].tag.urn).toEqual('urn:li:tag:unprimary-tag'); + expect(combinedData.dataset.globalTags.tags[1].tag.urn).toEqual('urn:li:tag:primary-tag'); + + // will overwrite string properties w/ primary + expect(combinedData.dataset.editableProperties.description).toEqual('secondary description'); + + // will take secondary string properties in the case of empty string + expect(combinedData.dataset.properties.description).toEqual('primary description'); + }); + }); + + describe('combineSiblingsInSearchResults', () => { + it('combines search results to deduplicate siblings', () => { + const result = combineSiblingsInSearchResults(searchResultWithSiblings as any); + + expect(result).toHaveLength(1); + expect(result?.[0]?.matchedEntities?.[0]?.urn).toEqual( + 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)', + ); + expect(result?.[0]?.matchedEntities?.[1]?.urn).toEqual( + 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)', + ); + }); + }); +}); diff --git a/datahub-web-react/src/app/entity/shared/components/legacy/Ownership.tsx b/datahub-web-react/src/app/entity/shared/components/legacy/Ownership.tsx deleted file mode 100644 index b5abf6e306df24..00000000000000 --- a/datahub-web-react/src/app/entity/shared/components/legacy/Ownership.tsx +++ /dev/null @@ -1,349 +0,0 @@ -import { AutoComplete, Button, Form, Select, Space, Table, Tag, Typography } from 'antd'; -import React, { useEffect, useMemo, useState } from 'react'; -import styled from 'styled-components'; -import { - CorpUser, - EntityType, - Owner, - OwnershipSourceType, - OwnershipType, - OwnershipUpdate, -} from '../../../../../types.generated'; -import CustomAvatar from '../../../../shared/avatar/CustomAvatar'; -import { useGetAutoCompleteResultsLazyQuery } from '../../../../../graphql/search.generated'; -import { useEntityRegistry } from '../../../../useEntityRegistry'; - -const UpdatedText = styled(Typography.Text)` - position: absolute; - right: 0; - margin: 0; -`; - -const OWNER_SEARCH_PLACEHOLDER = 'Search an LDAP'; -const NUMBER_OWNERS_REQUIRED = 2; - -interface Props { - owners: Array; - lastModifiedAt: number; - updateOwnership?: (update: OwnershipUpdate) => void; -} - -/** - * Displays an array of owners! Work-in-progress. - * - * TODO: Add mutations to change ownership on explicit save. - */ -export const Ownership: React.FC = ({ owners, lastModifiedAt, updateOwnership }: Props): JSX.Element => { - const entityRegistry = useEntityRegistry(); - - const [form] = Form.useForm(); - const [editingIndex, setEditingIndex] = useState(-1); - const [stagedOwners, setStagedOwners] = useState(owners); - const [ownerQuery, setOwnerQuery] = useState(''); - const [getOwnerAutoCompleteResults, { data: searchOwnerSuggestionsData }] = useGetAutoCompleteResultsLazyQuery(); - - useEffect(() => { - setStagedOwners(owners); - }, [owners]); - - const ownerTableData = useMemo( - () => - // eslint-disable-next-line consistent-return, array-callback-return - stagedOwners.map((owner, index) => { - if (owner.owner.__typename === 'CorpUser') { - return { - key: index, - urn: owner.owner.urn, - ldap: owner.owner.username, - fullName: owner.owner.info?.fullName || owner.owner.username, - role: owner.type, - pictureLink: owner.owner.editableInfo?.pictureLink, - type: EntityType.CorpUser, - }; - } - if (owner.owner.__typename === 'CorpGroup') { - return { - key: index, - urn: owner.owner.urn, - ldap: owner.owner.name, - fullName: owner.owner.name, - role: owner.type, - type: EntityType.CorpGroup, - }; - } - return { - key: index, - urn: owner.owner.urn, - ldap: (owner.owner as CorpUser).username, - fullName: (owner.owner as CorpUser).info?.fullName || (owner.owner as CorpUser).username, - role: owner.type, - pictureLink: (owner.owner as CorpUser).editableInfo?.pictureLink, - type: EntityType.CorpUser, - }; - }), - [stagedOwners], - ); - - const isEditing = (record: { key: number }) => record.key === editingIndex; - - const onAdd = () => { - setEditingIndex(stagedOwners.length); - - form.setFieldsValue({ - ldap: '', - role: OwnershipType.Stakeholder, - type: EntityType.CorpUser, - }); - - const newOwner = { - owner: { - type: EntityType.CorpUser, - urn: '', - username: '', - __typename: 'CorpUser' as const, - }, - type: OwnershipType.Stakeholder, - source: { - type: OwnershipSourceType.Manual, - }, - }; - - const newStagedOwners = [...stagedOwners, newOwner]; - setStagedOwners(newStagedOwners); - }; - - const onDelete = (urn: string, role: OwnershipType) => { - if (updateOwnership) { - const updatedOwners = owners - .filter((owner) => !(owner.owner.urn === urn && owner.type === role)) - .map((owner) => ({ - owner: owner.owner.urn, - type: owner.type, - })); - - updateOwnership({ owners: updatedOwners }); - } - }; - - const onChangeOwnerQuery = async (query: string) => { - if (query && query !== '') { - const row = await form.validateFields(); - getOwnerAutoCompleteResults({ - variables: { - input: { - type: row.type, - query, - field: row.type === EntityType.CorpUser ? 'ldap' : 'name', - }, - }, - }); - } - setOwnerQuery(query); - }; - - const onSave = async (record: any) => { - if (updateOwnership) { - const row = await form.validateFields(); - const updatedOwners = stagedOwners.map((owner, index) => { - if (record.key === index) { - return { - owner: `urn:li:${row.type === EntityType.CorpGroup ? 'corpGroup' : 'corpuser'}:${row.ldap}`, - type: row.role, - }; - } - return { - owner: owner.owner.urn, - type: owner.type, - }; - }); - updateOwnership({ owners: updatedOwners }); - } - setEditingIndex(-1); - }; - - const onCancel = () => { - const newStagedOwners = stagedOwners.filter((_, index) => index !== editingIndex); - setStagedOwners(newStagedOwners); - setEditingIndex(-1); - }; - - const onSelectSuggestion = (ldap: string) => { - setOwnerQuery(ldap); - }; - - const ownerTableColumns = [ - { - title: 'LDAP', - dataIndex: 'ldap', - render: (text: string, record: any) => { - return isEditing(record) ? ( - - ({ - value: suggestion, - }))) || - [] - } - value={ownerQuery} - onSelect={onSelectSuggestion} - onSearch={onChangeOwnerQuery} - placeholder={OWNER_SEARCH_PLACEHOLDER} - /> - - ) : ( - - ); - }, - }, - { - title: 'Full Name', - dataIndex: 'fullName', - }, - { - title: 'Role', - dataIndex: 'role', - render: (role: OwnershipType, record: any) => { - return isEditing(record) ? ( - - - - ) : ( - {role} - ); - }, - }, - { - title: 'Type', - dataIndex: 'type', - render: (type: EntityType, record: any) => { - return isEditing(record) ? ( - - - - ) : ( - {type} - ); - }, - }, - { - title: '', - key: 'action', - render: (_: string, record: any) => { - return ( - - {isEditing(record) ? ( - <> - - - - ) : ( - - )} - - ); - }, - }, - ]; - - return ( - <> - {!!lastModifiedAt && ( - - Last updated {new Date(lastModifiedAt).toLocaleDateString('en-US')} - - )} - - Ownership - - Please maintain at least {NUMBER_OWNERS_REQUIRED} owners. - -
- - - {editingIndex < 0 && ( - - )} - - - ); -}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx index 3b982fea2ac358..ea30042a5bbeef 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx @@ -2,7 +2,7 @@ import React, { useState } from 'react'; import { message, Modal, Button, Form, Input } from 'antd'; import { PlusOutlined } from '@ant-design/icons'; import { useGetAuthenticatedUser } from '../../../../useGetAuthenticatedUser'; -import { useEntityData } from '../../EntityContext'; +import { useEntityData, useMutationUrn } from '../../EntityContext'; import { useAddLinkMutation } from '../../../../../graphql/mutations.generated'; import analytics, { EventType, EntityActionType } from '../../../../analytics'; @@ -13,8 +13,9 @@ type AddLinkProps = { export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => { const [isModalVisible, setIsModalVisible] = useState(false); + const mutationUrn = useMutationUrn(); const user = useGetAuthenticatedUser(); - const { urn, entityType } = useEntityData(); + const { entityType } = useEntityData(); const [addLinkMutation] = useAddLinkMutation(); const [form] = Form.useForm(); @@ -32,13 +33,13 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => { if (user?.corpUser.urn) { try { await addLinkMutation({ - variables: { input: { linkUrl: formData.url, label: formData.label, resourceUrn: urn } }, + variables: { input: { linkUrl: formData.url, label: formData.label, resourceUrn: mutationUrn } }, }); message.success({ content: 'Link Added', duration: 2 }); analytics.event({ type: EventType.EntityActionEvent, entityType, - entityUrn: urn, + entityUrn: mutationUrn, actionType: EntityActionType.UpdateLinks, }); } catch (e: unknown) { diff --git a/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx new file mode 100644 index 00000000000000..7bd19066758b88 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx @@ -0,0 +1,112 @@ +import { InfoCircleOutlined } from '@ant-design/icons'; +import { Divider, Popover, Tooltip, Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import moment from 'moment'; +import { Deprecation } from '../../../../../types.generated'; +import { getLocaleTimezone } from '../../../../shared/time/timeUtils'; +import { ANTD_GRAY } from '../../constants'; + +const DeprecatedContainer = styled.div` + width: 104px; + height: 18px; + border: 1px solid #ef5b5b; + border-radius: 15px; + display: flex; + justify-content: center; + align-items: center; + color: #ef5b5b; + margin-left: 0px; + padding-top: 12px; + padding-bottom: 12px; +`; + +const DeprecatedText = styled.div` + color: #ef5b5b; + margin-left: 5px; +`; + +const DeprecatedTitle = styled(Typography.Text)` + display: block; + font-size: 14px; + margin-bottom: 5px; + font-weight: bold; +`; + +const DeprecatedSubTitle = styled(Typography.Text)` + display: block; + margin-bottom: 5px; +`; + +const LastEvaluatedAtLabel = styled.div` + padding: 0; + margin: 0; + display: flex; + align-items: center; + color: ${ANTD_GRAY[7]}; +`; + +const ThinDivider = styled(Divider)` + margin-top: 8px; + margin-bottom: 8px; +`; + +const StyledInfoCircleOutlined = styled(InfoCircleOutlined)` + color: #ef5b5b; +`; + +type Props = { + deprecation: Deprecation; + preview?: boolean | null; +}; + +export const DeprecationPill = ({ deprecation, preview }: Props) => { + /** + * Deprecation Decommission Timestamp + */ + const localeTimezone = getLocaleTimezone(); + const decommissionTimeLocal = + (deprecation.decommissionTime && + `Scheduled to be decommissioned on ${moment + .unix(deprecation.decommissionTime) + .format('DD/MMM/YYYY')} (${localeTimezone})`) || + undefined; + const decommissionTimeGMT = + deprecation.decommissionTime && + moment.unix(deprecation.decommissionTime).utc().format('dddd, DD/MMM/YYYY HH:mm:ss z'); + + const hasDetails = deprecation.note !== '' || deprecation.decommissionTime !== null; + const isDividerNeeded = deprecation.note !== '' && deprecation.decommissionTime !== null; + + return ( + + {deprecation?.note !== '' && Deprecation note} + {isDividerNeeded && } + {deprecation?.note !== '' && {deprecation.note}} + {deprecation?.decommissionTime !== null && ( + + + {decommissionTimeLocal} + + + )} + + ) : ( + 'No additional details' + ) + } + > + {(preview && ) || ( + + + Deprecated + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ExpandedActor.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ExpandedActor.tsx new file mode 100644 index 00000000000000..057b4c6706f251 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/ExpandedActor.tsx @@ -0,0 +1,49 @@ +import { Popover, Tag } from 'antd'; +import React from 'react'; +import { Link } from 'react-router-dom'; +import styled from 'styled-components'; +import { CorpGroup, CorpUser, EntityType } from '../../../../../types.generated'; +import { CustomAvatar } from '../../../../shared/avatar'; +import { useEntityRegistry } from '../../../../useEntityRegistry'; + +type Props = { + actor: CorpUser | CorpGroup; + popOver?: React.ReactNode; + closable?: boolean | undefined; + onClose?: () => void; +}; + +const ActorTag = styled(Tag)` + padding: 2px; + padding-right: 6px; + margin-bottom: 8px; + display: inline-flex; + align-items: center; +`; + +export const ExpandedActor = ({ actor, popOver, closable, onClose }: Props) => { + const entityRegistry = useEntityRegistry(); + + let name = ''; + if (actor.__typename === 'CorpGroup') { + name = entityRegistry.getDisplayName(EntityType.CorpGroup, actor); + } + if (actor.__typename === 'CorpUser') { + name = entityRegistry.getDisplayName(EntityType.CorpUser, actor); + } + + const pictureLink = (actor.__typename === 'CorpUser' && actor.editableProperties?.pictureLink) || undefined; + + return ( + + + + {(!popOver && <>{name}) || ( + + {name} + + )} + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ExpandedActorGroup.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ExpandedActorGroup.tsx new file mode 100644 index 00000000000000..887e0f2421cbdf --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/ExpandedActorGroup.tsx @@ -0,0 +1,54 @@ +import { Popover, Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import { CorpGroup, CorpUser } from '../../../../../types.generated'; +import { ExpandedActor } from './ExpandedActor'; + +const PopoverActors = styled.div``; + +const ActorsContainer = styled.div` + display: flex; + justify-content: right; + flex-wrap: wrap; + align-items: center; +`; + +const RemainderText = styled(Typography.Text)` + display: flex; + justify-content: right; + margin-right: 8px; +`; + +type Props = { + actors: Array; + max: number; + onClose?: (actor: CorpUser | CorpGroup) => void; + containerStyle?: any; +}; + +const DEFAULT_MAX = 10; + +export const ExpandedActorGroup = ({ actors, max = DEFAULT_MAX, onClose, containerStyle }: Props) => { + const finalActors = actors.length > max ? actors.slice(0, max) : actors; + const remainder = actors.length > max ? actors.length - max : undefined; + + return ( + + {actors.map((actor) => ( + onClose?.(actor)} /> + ))} + + } + > + + {finalActors.map((actor) => ( + onClose?.(actor)} /> + ))} + + {remainder && + {remainder} more} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/ExpandedOwner.tsx b/datahub-web-react/src/app/entity/shared/components/styled/ExpandedOwner.tsx index f79da080a449d1..41f445f9addf43 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/ExpandedOwner.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/ExpandedOwner.tsx @@ -12,7 +12,7 @@ import { useEntityData } from '../../EntityContext'; import { getDescriptionFromType, getNameFromType } from '../../containers/profile/sidebar/Ownership/ownershipUtils'; type Props = { - entityUrn: string; + entityUrn?: string; owner: Owner; hidePopOver?: boolean | undefined; refetch?: () => Promise; @@ -43,6 +43,9 @@ export const ExpandedOwner = ({ entityUrn, owner, hidePopOver, refetch }: Props) (owner.owner.__typename === 'CorpUser' && owner.owner.editableProperties?.pictureLink) || undefined; const onDelete = async () => { + if (!entityUrn) { + return; + } try { await removeOwnerMutation({ variables: { @@ -84,7 +87,7 @@ export const ExpandedOwner = ({ entityUrn, owner, hidePopOver, refetch }: Props) }; return ( - + {(hidePopOver && <>{name}) || ( diff --git a/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx b/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx new file mode 100644 index 00000000000000..8146fa62bc2195 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/StatsSummary.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import styled from 'styled-components'; +import { ANTD_GRAY } from '../../constants'; + +type Props = { + stats: Array; +}; + +const StatsContainer = styled.div` + margin-top: 8px; +`; + +const StatDivider = styled.div` + display: inline-block; + padding-left: 10px; + margin-right: 10px; + border-right: 1px solid ${ANTD_GRAY[4]}; + height: 21px; + vertical-align: text-top; +`; + +export const StatsSummary = ({ stats }: Props) => { + return ( + <> + {stats && stats.length > 0 && ( + + {stats.map((statView, index) => ( + + {statView} + {index < stats.length - 1 && } + + ))} + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx index 0d497ea7db2316..38b7f48215df94 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx @@ -1,25 +1,22 @@ -import React, { useState } from 'react'; -import * as QueryString from 'query-string'; -import { useHistory, useLocation, useParams } from 'react-router'; +import React, { useState, useEffect } from 'react'; import { message } from 'antd'; import styled from 'styled-components'; import { ApolloError } from '@apollo/client'; - -import { useEntityRegistry } from '../../../../../useEntityRegistry'; import { EntityType, FacetFilterInput } from '../../../../../../types.generated'; -import useFilters from '../../../../../search/utils/useFilters'; import { ENTITY_FILTER_NAME } from '../../../../../search/utils/constants'; import { SearchCfg } from '../../../../../../conf'; -import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl'; import { EmbeddedListSearchResults } from './EmbeddedListSearchResults'; import EmbeddedListSearchHeader from './EmbeddedListSearchHeader'; import { useGetSearchResultsForMultipleQuery } from '../../../../../../graphql/search.generated'; import { GetSearchResultsParams, SearchResultsInterface } from './types'; +import { isListSubset } from '../../../utils'; +import { EntityAndType } from '../../../types'; const Container = styled.div` display: flex; flex-direction: column; height: 100%; + overflow-y: hidden; `; // this extracts the response from useGetSearchResultsForMultipleQuery into a common interface other search endpoints can also produce @@ -48,15 +45,21 @@ export const addFixedQuery = (baseQuery: string, fixedQuery: string, emptyQuery: return finalQuery; }; -type SearchPageParams = { - type?: string; -}; - type Props = { + query: string; + page: number; + filters: FacetFilterInput[]; + onChangeQuery: (query) => void; + onChangeFilters: (filters) => void; + onChangePage: (page) => void; emptySearchQuery?: string | null; fixedFilter?: FacetFilterInput | null; fixedQuery?: string | null; placeholderText?: string | null; + defaultShowFilters?: boolean; + defaultFilters?: Array; + searchBarStyle?: any; + searchBarInputStyle?: any; useGetSearchResults?: (params: GetSearchResultsParams) => { data: SearchResultsInterface | undefined | null; loading: boolean; @@ -66,21 +69,26 @@ type Props = { }; export const EmbeddedListSearch = ({ + query, + filters, + page, + onChangeQuery, + onChangeFilters, + onChangePage, emptySearchQuery, fixedFilter, fixedQuery, placeholderText, + defaultShowFilters, + defaultFilters, + searchBarStyle, + searchBarInputStyle, useGetSearchResults = useWrappedSearchResults, }: Props) => { - const history = useHistory(); - const location = useLocation(); - const entityRegistry = useEntityRegistry(); - - const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); - const query: string = addFixedQuery(params?.query as string, fixedQuery as string, emptySearchQuery as string); - const activeType = entityRegistry.getTypeOrDefaultFromPathName(useParams().type || '', undefined); - const page: number = params.page && Number(params.page as string) > 0 ? Number(params.page as string) : 1; - const filters: Array = useFilters(params); + // Adjust query based on props + const finalQuery: string = addFixedQuery(query as string, fixedQuery as string, emptySearchQuery as string); + + // Adjust filters based on props const filtersWithoutEntities: Array = filters.filter( (filter) => filter.field !== ENTITY_FILTER_NAME, ); @@ -89,13 +97,16 @@ export const EmbeddedListSearch = ({ .filter((filter) => filter.field === ENTITY_FILTER_NAME) .map((filter) => filter.value.toUpperCase() as EntityType); - const [showFilters, setShowFilters] = useState(false); + const [showFilters, setShowFilters] = useState(defaultShowFilters || false); + const [isSelectMode, setIsSelectMode] = useState(false); + const [selectedEntities, setSelectedEntities] = useState([]); + const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE); - const { refetch } = useGetSearchResults({ + const { refetch: refetchForDownload } = useGetSearchResults({ variables: { input: { types: entityFilters, - query, + query: finalQuery, start: (page - 1) * SearchCfg.RESULTS_PER_PAGE, count: SearchCfg.RESULTS_PER_PAGE, filters: finalFilters, @@ -105,57 +116,65 @@ export const EmbeddedListSearch = ({ }); const callSearchOnVariables = (variables: GetSearchResultsParams['variables']) => { - return refetch(variables); + return refetchForDownload(variables); }; - const { data, loading, error } = useGetSearchResults({ + const { data, loading, error, refetch } = useGetSearchResults({ variables: { input: { types: entityFilters, - query, - start: (page - 1) * SearchCfg.RESULTS_PER_PAGE, - count: SearchCfg.RESULTS_PER_PAGE, + query: finalQuery, + start: (page - 1) * numResultsPerPage, + count: numResultsPerPage, filters: finalFilters, }, }, }); - const onSearch = (q: string) => { - const finalQuery = addFixedQuery(q as string, fixedQuery as string, emptySearchQuery as string); - navigateToEntitySearchUrl({ - baseUrl: location.pathname, - type: activeType, - query: finalQuery, - page: 1, - history, - }); - }; + const searchResultEntities = + data?.searchResults?.map((result) => ({ urn: result.entity.urn, type: result.entity.type })) || []; + const searchResultUrns = searchResultEntities.map((entity) => entity.urn); + const selectedEntityUrns = selectedEntities.map((entity) => entity.urn); - const onChangeFilters = (newFilters: Array) => { - navigateToEntitySearchUrl({ - baseUrl: location.pathname, - type: activeType, - query, - page: 1, - filters: newFilters, - history, - }); + const onToggleFilters = () => { + setShowFilters(!showFilters); }; - const onChangePage = (newPage: number) => { - navigateToEntitySearchUrl({ - baseUrl: location.pathname, - type: activeType, - query, - page: newPage, - filters, - history, - }); + /** + * Invoked when the "select all" checkbox is clicked. + * + * This method either adds the entire current page of search results to + * the list of selected entities, or removes the current page from the set of selected entities. + */ + const onChangeSelectAll = (selected: boolean) => { + if (selected) { + // Add current page of urns to the master selected entity list + const entitiesToAdd = searchResultEntities.filter( + (entity) => + selectedEntities.findIndex( + (element) => element.urn === entity.urn && element.type === entity.type, + ) < 0, + ); + setSelectedEntities(Array.from(new Set(selectedEntities.concat(entitiesToAdd)))); + } else { + // Filter out the current page of entity urns from the list + setSelectedEntities(selectedEntities.filter((entity) => searchResultUrns.indexOf(entity.urn) === -1)); + } }; - const toggleFilters = () => { - setShowFilters(!showFilters); - }; + useEffect(() => { + if (!isSelectMode) { + setSelectedEntities([]); + } + }, [isSelectMode]); + + useEffect(() => { + if (defaultFilters) { + onChangeFilters(defaultFilters); + } + // only want to run once on page load + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); // Filter out the persistent filter values const filteredFilters = data?.facets?.filter((facet) => facet.field !== fixedFilter?.field) || []; @@ -164,14 +183,21 @@ export const EmbeddedListSearch = ({ {error && message.error(`Failed to complete search: ${error && error.message}`)} onChangeQuery(addFixedQuery(q, fixedQuery as string, emptySearchQuery as string))} placeholderText={placeholderText} - onToggleFilters={toggleFilters} - showDownloadCsvButton + onToggleFilters={onToggleFilters} callSearchOnVariables={callSearchOnVariables} entityFilters={entityFilters} filters={finalFilters} - query={query} + query={finalQuery} + isSelectMode={isSelectMode} + isSelectAll={selectedEntities.length > 0 && isListSubset(searchResultUrns, selectedEntityUrns)} + setIsSelectMode={setIsSelectMode} + selectedEntities={selectedEntities} + onChangeSelectAll={onChangeSelectAll} + refetch={refetch as any} + searchBarStyle={searchBarStyle} + searchBarInputStyle={searchBarInputStyle} /> ); diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx index 9dfe3a93b1ff9f..0b9f1ff67510a6 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx @@ -8,20 +8,23 @@ import { useEntityRegistry } from '../../../../../useEntityRegistry'; import { EntityType, FacetFilterInput, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; import { SearchResultsInterface } from './types'; import SearchExtendedMenu from './SearchExtendedMenu'; +import { SearchSelectBar } from './SearchSelectBar'; +import { EntityAndType } from '../../../types'; const HeaderContainer = styled.div` display: flex; justify-content: space-between; - padding-bottom: 16px; width: 100%; + padding-right: 4px; + padding-left: 4px; `; const SearchAndDownloadContainer = styled.div` display: flex; + align-items: center; `; const SearchMenuContainer = styled.div` - margin-top: 7px; margin-left: 10px; `; @@ -29,68 +32,95 @@ type Props = { onSearch: (q: string) => void; onToggleFilters: () => void; placeholderText?: string | null; - showDownloadCsvButton?: boolean; callSearchOnVariables: (variables: { input: SearchAcrossEntitiesInput; }) => Promise; entityFilters: EntityType[]; filters: FacetFilterInput[]; query: string; + isSelectMode: boolean; + isSelectAll: boolean; + selectedEntities: EntityAndType[]; + setIsSelectMode: (showSelectMode: boolean) => any; + onChangeSelectAll: (selected: boolean) => void; + refetch?: () => void; + searchBarStyle?: any; + searchBarInputStyle?: any; }; export default function EmbeddedListSearchHeader({ onSearch, onToggleFilters, placeholderText, - showDownloadCsvButton, callSearchOnVariables, entityFilters, filters, query, + isSelectMode, + isSelectAll, + selectedEntities, + setIsSelectMode, + onChangeSelectAll, + refetch, + searchBarStyle, + searchBarInputStyle, }: Props) { const entityRegistry = useEntityRegistry(); - const onQueryChange = (newQuery: string) => { - onSearch(newQuery); - }; - return ( - - - - - - {/* TODO: in the future, when we add more menu items, we'll show this always */} - {showDownloadCsvButton && ( + <> + + + + + - )} - - - + + + + {isSelectMode && ( + + { + setIsSelectMode(false); + }} + refetch={refetch} + /> + + )} + ); } diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx new file mode 100644 index 00000000000000..4293789fcdd0b0 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchModal.tsx @@ -0,0 +1,88 @@ +import React, { useState } from 'react'; +import { Button, Modal } from 'antd'; +import styled from 'styled-components'; +import { FacetFilterInput } from '../../../../../../types.generated'; +import { EmbeddedListSearch } from './EmbeddedListSearch'; + +const SearchContainer = styled.div` + height: 500px; +`; +const modalStyle = { + top: 40, +}; + +const modalBodyStyle = { + padding: 0, +}; + +type Props = { + emptySearchQuery?: string | null; + fixedFilter?: FacetFilterInput | null; + fixedQuery?: string | null; + placeholderText?: string | null; + defaultShowFilters?: boolean; + defaultFilters?: Array; + onClose?: () => void; + searchBarStyle?: any; + searchBarInputStyle?: any; +}; + +export const EmbeddedListSearchModal = ({ + emptySearchQuery, + fixedFilter, + fixedQuery, + placeholderText, + defaultShowFilters, + defaultFilters, + onClose, + searchBarStyle, + searchBarInputStyle, +}: Props) => { + // Component state + const [query, setQuery] = useState(''); + const [page, setPage] = useState(1); + const [filters, setFilters] = useState>([]); + + const onChangeQuery = (q: string) => { + setQuery(q); + }; + + const onChangeFilters = (newFilters: Array) => { + setFilters(newFilters); + }; + + const onChangePage = (newPage: number) => { + setPage(newPage); + }; + + return ( + Close} + > + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx index 50c1d957881ef4..6fc3bc3bd9e984 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchResults.tsx @@ -6,12 +6,12 @@ import { SearchFilters } from '../../../../../search/SearchFilters'; import { SearchCfg } from '../../../../../../conf'; import { EntityNameList } from '../../../../../recommendations/renderer/component/EntityNameList'; import { ReactComponent as LoadingSvg } from '../../../../../../images/datahub-logo-color-loading_pendulum.svg'; +import { EntityAndType } from '../../../types'; const SearchBody = styled.div` + height: 100%; + overflow-y: scroll; display: flex; - flex-direction: row; - flex: 1 1 auto; - overflow-y: hidden; `; const PaginationInfo = styled(Typography.Text)` @@ -29,16 +29,15 @@ const FiltersContainer = styled.div` `; const ResultContainer = styled.div` - flex: 1; + height: auto; overflow: auto; - display: flex; - flex-direction: column; + flex: 1; `; -const PaginationInfoContainer = styled.div` +const PaginationInfoContainer = styled.span` padding: 8px; padding-left: 16px; - border-bottom: 1px solid; + border-top: 1px solid; border-color: ${(props) => props.theme.styles['border-color-base']}; display: flex; justify-content: space-between; @@ -72,11 +71,6 @@ const SearchFilterContainer = styled.div` overflow: hidden; `; -const LoadingText = styled.div` - margin-top: 18px; - font-size: 12px; -`; - const LoadingContainer = styled.div` padding-top: 40px; padding-bottom: 40px; @@ -94,6 +88,11 @@ interface Props { showFilters?: boolean; onChangeFilters: (filters: Array) => void; onChangePage: (page: number) => void; + isSelectMode: boolean; + selectedEntities: EntityAndType[]; + setSelectedEntities: (entities: EntityAndType[]) => any; + numResultsPerPage: number; + setNumResultsPerPage: (numResults: number) => void; } export const EmbeddedListSearchResults = ({ @@ -105,16 +104,17 @@ export const EmbeddedListSearchResults = ({ showFilters, onChangeFilters, onChangePage, + isSelectMode, + selectedEntities, + setSelectedEntities, + numResultsPerPage, + setNumResultsPerPage, }: Props) => { const pageStart = searchResponse?.start || 0; const pageSize = searchResponse?.count || 0; const totalResults = searchResponse?.total || 0; const lastResultIndex = pageStart + pageSize > totalResults ? totalResults : pageStart + pageSize; - const onFilterSelect = (newFilters) => { - onChangeFilters(newFilters); - }; - return ( <> @@ -126,7 +126,7 @@ export const EmbeddedListSearchResults = ({ loading={loading} facets={filters || []} selectedFilters={selectedFilters} - onFilterSelect={onFilterSelect} + onFilterSelect={(newFilters) => onChangeFilters(newFilters)} /> @@ -135,44 +135,45 @@ export const EmbeddedListSearchResults = ({ {loading && ( - Searching for related entities... )} {!loading && ( - <> - searchResult.entity) || [] - } - additionalPropertiesList={ - searchResponse?.searchResults?.map((searchResult) => ({ - // when we add impact analysis, we will want to pipe the path to each element to the result this - // eslint-disable-next-line @typescript-eslint/dot-notation - degree: searchResult['degree'], - })) || [] - } - /> - - )} - - - - {lastResultIndex > 0 ? (page - 1) * pageSize + 1 : 0} - {lastResultIndex} - {' '} - of {totalResults} - - searchResult.entity) || []} + additionalPropertiesList={ + searchResponse?.searchResults?.map((searchResult) => ({ + // when we add impact analysis, we will want to pipe the path to each element to the result this + // eslint-disable-next-line @typescript-eslint/dot-notation + degree: searchResult['degree'], + })) || [] + } + isSelectMode={isSelectMode} + selectedEntities={selectedEntities} + setSelectedEntities={setSelectedEntities} + bordered={false} /> - - + )} + + + + {lastResultIndex > 0 ? (page - 1) * pageSize + 1 : 0} - {lastResultIndex} + {' '} + of {totalResults} + + SearchCfg.RESULTS_PER_PAGE} + onShowSizeChange={(_currNum, newNum) => setNumResultsPerPage(newNum)} + pageSizeOptions={['10', '20', '50', '100']} + /> + + ); }; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx new file mode 100644 index 00000000000000..c121775c6e9aa1 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchSection.tsx @@ -0,0 +1,100 @@ +import React from 'react'; +import * as QueryString from 'query-string'; +import { useHistory, useLocation } from 'react-router'; +import { ApolloError } from '@apollo/client'; +import { FacetFilterInput } from '../../../../../../types.generated'; +import useFilters from '../../../../../search/utils/useFilters'; +import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl'; +import { GetSearchResultsParams, SearchResultsInterface } from './types'; +import { useEntityQueryParams } from '../../../containers/profile/utils'; +import { EmbeddedListSearch } from './EmbeddedListSearch'; + +type Props = { + emptySearchQuery?: string | null; + fixedFilter?: FacetFilterInput | null; + fixedQuery?: string | null; + placeholderText?: string | null; + defaultShowFilters?: boolean; + defaultFilters?: Array; + searchBarStyle?: any; + searchBarInputStyle?: any; + useGetSearchResults?: (params: GetSearchResultsParams) => { + data: SearchResultsInterface | undefined | null; + loading: boolean; + error: ApolloError | undefined; + refetch: (variables: GetSearchResultsParams['variables']) => Promise; + }; +}; + +export const EmbeddedListSearchSection = ({ + emptySearchQuery, + fixedFilter, + fixedQuery, + placeholderText, + defaultShowFilters, + defaultFilters, + searchBarStyle, + searchBarInputStyle, + useGetSearchResults, +}: Props) => { + const history = useHistory(); + const location = useLocation(); + const baseParams = useEntityQueryParams(); + + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const query: string = params?.query as string; + const page: number = params.page && Number(params.page as string) > 0 ? Number(params.page as string) : 1; + const filters: Array = useFilters(params); + + const onSearch = (q: string) => { + navigateToEntitySearchUrl({ + baseUrl: location.pathname, + baseParams, + query: q, + page: 1, + history, + }); + }; + + const onChangeFilters = (newFilters: Array) => { + navigateToEntitySearchUrl({ + baseUrl: location.pathname, + baseParams, + query, + page: 1, + filters: newFilters, + history, + }); + }; + + const onChangePage = (newPage: number) => { + navigateToEntitySearchUrl({ + baseUrl: location.pathname, + baseParams, + query, + page: newPage, + filters, + history, + }); + }; + + return ( + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx index 6e453ae62d5cd7..7dd6caa291b97c 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx @@ -1,6 +1,6 @@ import React, { useState } from 'react'; -import { Dropdown, Menu } from 'antd'; -import { MoreOutlined } from '@ant-design/icons'; +import { Button, Dropdown, Menu } from 'antd'; +import { FormOutlined, MoreOutlined } from '@ant-design/icons'; import styled from 'styled-components'; import { EntityType, FacetFilterInput, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; import { SearchResultsInterface } from './types'; @@ -8,10 +8,20 @@ import DownloadAsCsvButton from './DownloadAsCsvButton'; import DownloadAsCsvModal from './DownloadAsCsvModal'; const MenuIcon = styled(MoreOutlined)` - font-size: 15px; + font-size: 20px; height: 20px; `; +const SelectButton = styled(Button)` + font-size: 12px; + padding-left: 12px; + padding-right: 12px; +`; + +const MenuItem = styled(Menu.Item)` + padding: 0px; +`; + type Props = { callSearchOnVariables: (variables: { input: SearchAcrossEntitiesInput; @@ -19,21 +29,36 @@ type Props = { entityFilters: EntityType[]; filters: FacetFilterInput[]; query: string; + setShowSelectMode?: (showSelectMode: boolean) => any; }; // currently only contains Download As Csv but will be extended to contain other actions as well -export default function SearchExtendedMenu({ callSearchOnVariables, entityFilters, filters, query }: Props) { +export default function SearchExtendedMenu({ + callSearchOnVariables, + entityFilters, + filters, + query, + setShowSelectMode, +}: Props) { const [isDownloadingCsv, setIsDownloadingCsv] = useState(false); const [showDownloadAsCsvModal, setShowDownloadAsCsvModal] = useState(false); const menu = ( - + - + + {setShowSelectMode && ( + + setShowSelectMode(true)}> + + Edit... + + + )} ); diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelect.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelect.tsx new file mode 100644 index 00000000000000..903e385c08477c --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelect.tsx @@ -0,0 +1,178 @@ +import React, { useState } from 'react'; +import { Button, message, Typography } from 'antd'; +import styled from 'styled-components'; +import { FilterOutlined } from '@ant-design/icons'; + +import { useEntityRegistry } from '../../../../../useEntityRegistry'; +import { EntityType, FacetFilterInput } from '../../../../../../types.generated'; +import { ENTITY_FILTER_NAME } from '../../../../../search/utils/constants'; +import { SearchCfg } from '../../../../../../conf'; +import { EmbeddedListSearchResults } from './EmbeddedListSearchResults'; +import { useGetSearchResultsForMultipleQuery } from '../../../../../../graphql/search.generated'; +import { isListSubset } from '../../../utils'; +import { SearchBar } from '../../../../../search/SearchBar'; +import { ANTD_GRAY } from '../../../constants'; +import { EntityAndType } from '../../../types'; +import { SearchSelectBar } from './SearchSelectBar'; +import TabToolbar from '../TabToolbar'; + +const Container = styled.span` + display: flex; + flex-direction: column; + height: 100%; +`; + +const SearchBarContainer = styled.div` + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px; + border-bottom: 1px solid ${ANTD_GRAY[4]}; +`; + +const SEARCH_BAR_STYLE = { + maxWidth: 680, + padding: 0, +}; + +const SEARCH_INPUT_STYLE = { + height: 40, + fontSize: 12, +}; + +type Props = { + fixedEntityTypes?: Array | null; + placeholderText?: string | null; + selectedEntities: EntityAndType[]; + setSelectedEntities: (Entities: EntityAndType[]) => void; +}; + +/** + * An embeddable component that can be used for searching & selecting a subset of the entities on the Metadata Graph + * in order to perform some action. + * + * This component provides easy ways to filter for a specific set of entity types, and provides a set of entity urns + * when the selection is complete. + */ +export const SearchSelect = ({ fixedEntityTypes, placeholderText, selectedEntities, setSelectedEntities }: Props) => { + const entityRegistry = useEntityRegistry(); + + // Component state + const [query, setQuery] = useState(''); + const [page, setPage] = useState(1); + const [filters, setFilters] = useState>([]); + const [showFilters, setShowFilters] = useState(false); + const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE); + + // Compute search filters + const entityFilters: Array = filters + .filter((filter) => filter.field === ENTITY_FILTER_NAME) + .map((filter) => filter.value.toUpperCase() as EntityType); + const finalEntityTypes = (entityFilters.length > 0 && entityFilters) || fixedEntityTypes || []; + + // Execute search + const { data, loading, error, refetch } = useGetSearchResultsForMultipleQuery({ + variables: { + input: { + types: finalEntityTypes, + query, + start: (page - 1) * numResultsPerPage, + count: numResultsPerPage, + filters, + }, + }, + }); + + const searchAcrossEntities = data?.searchAcrossEntities; + const searchResultEntities = + searchAcrossEntities?.searchResults?.map((result) => ({ urn: result.entity.urn, type: result.entity.type })) || + []; + const searchResultUrns = searchResultEntities.map((entity) => entity.urn); + const selectedEntityUrns = selectedEntities.map((entity) => entity.urn); + const facets = searchAcrossEntities?.facets || []; + + const onSearch = (q: string) => { + setQuery(q); + }; + + const onChangeFilters = (newFilters: Array) => { + setFilters(newFilters); + }; + + const onChangePage = (newPage: number) => { + setPage(newPage); + }; + + const onToggleFilters = () => { + setShowFilters(!showFilters); + }; + + /** + * Invoked when the "select all" checkbox is clicked. + * + * This method either adds the entire current page of search results to + * the list of selected entities, or removes the current page from the set of selected entities. + */ + const onChangeSelectAll = (selected: boolean) => { + if (selected) { + // Add current page of urns to the master selected entity list + const entitiesToAdd = searchResultEntities.filter( + (entity) => + selectedEntities.findIndex( + (element) => element.urn === entity.urn && element.type === entity.type, + ) < 0, + ); + setSelectedEntities(Array.from(new Set(selectedEntities.concat(entitiesToAdd)))); + } else { + // Filter out the current page of entity urns from the list + setSelectedEntities(selectedEntities.filter((entity) => searchResultUrns.indexOf(entity.urn) === -1)); + } + }; + + return ( + + {error && message.error(`Failed to complete search: ${error && error.message}`)} + + + + + + 0 && isListSubset(searchResultUrns, selectedEntityUrns)} + onChangeSelectAll={onChangeSelectAll} + showCancel={false} + showActions={false} + refetch={refetch} + selectedEntities={selectedEntities} + /> + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectActions.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectActions.tsx new file mode 100644 index 00000000000000..f70907a9ad7fa1 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectActions.tsx @@ -0,0 +1,125 @@ +import React from 'react'; +import OwnersDropdown from './action/OwnersDropdown'; +import GlossaryTermDropdown from './action/GlossaryTermsDropdown'; +import TagsDropdown from './action/TagsDropdown'; +import DomainDropdown from './action/DomainsDropdown'; +import DeprecationDropdown from './action/DeprecationDropdown'; +import DeleteDropdown from './action/DeleteDropdown'; +import { EntityType } from '../../../../../../types.generated'; +import { EntityCapabilityType } from '../../../../Entity'; +import { useEntityRegistry } from '../../../../../useEntityRegistry'; +import { EntityAndType } from '../../../types'; +import { SelectActionGroups } from './types'; + +/** + * The set of action groups that are visible by default. + * + * Currently, only the change tags action is implemented. + */ +const DEFAULT_ACTION_GROUPS = [ + SelectActionGroups.CHANGE_TAGS, + SelectActionGroups.CHANGE_GLOSSARY_TERMS, + SelectActionGroups.CHANGE_DOMAINS, + SelectActionGroups.CHANGE_OWNERS, + SelectActionGroups.CHANGE_DEPRECATION, + SelectActionGroups.DELETE, +]; + +type Props = { + selectedEntities: EntityAndType[]; + visibleActionGroups?: Set; + refetch?: () => void; +}; + +/** + * A component used for rendering a group of actions to take on a group of selected entities such + * as changing owners, tags, domains, etc. + */ +export const SearchSelectActions = ({ + selectedEntities, + visibleActionGroups = new Set(DEFAULT_ACTION_GROUPS), + refetch, +}: Props) => { + const entityRegistry = useEntityRegistry(); + + /** + * Extract the urns and entity types, which are used for a) qualifying actions + * and b) executing actions. + */ + const selectedEntityUrns = selectedEntities.map((entity) => entity.urn); + const selectedEntityTypes = new Set(selectedEntities.map((entity) => entity.type)); + + /** + * Returns true if a specific capability is supported by ALL entities in a set. + */ + const isEntityCapabilitySupported = (type: EntityCapabilityType, entityTypes: Set) => { + return Array.from(entityTypes).every((entityType) => + entityRegistry.getSupportedEntityCapabilities(entityType).has(type), + ); + }; + + return ( + <> + {visibleActionGroups.has(SelectActionGroups.CHANGE_OWNERS) && ( + + )} + {visibleActionGroups.has(SelectActionGroups.CHANGE_GLOSSARY_TERMS) && ( + + )} + {visibleActionGroups.has(SelectActionGroups.CHANGE_TAGS) && ( + + )} + {visibleActionGroups.has(SelectActionGroups.CHANGE_DOMAINS) && ( + + )} + {visibleActionGroups.has(SelectActionGroups.CHANGE_DEPRECATION) && ( + + )} + {visibleActionGroups.has(SelectActionGroups.DELETE) && ( + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectBar.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectBar.tsx new file mode 100644 index 00000000000000..37bab2ef8fb6bd --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectBar.tsx @@ -0,0 +1,96 @@ +import React from 'react'; +import { Button, Checkbox, Modal, Typography } from 'antd'; +import styled from 'styled-components'; +import { ANTD_GRAY } from '../../../constants'; +import { EntityAndType } from '../../../types'; +import { SearchSelectActions } from './SearchSelectActions'; + +const CheckboxContainer = styled.div` + display: flex; + justify-content: left; + align-items: center; +`; + +const ActionsContainer = styled.div` + display: flex; + align-items: center; +`; + +const CancelButton = styled(Button)` + && { + margin-left: 8px; + padding: 0px; + color: ${ANTD_GRAY[7]}; + } +`; + +const StyledCheckbox = styled(Checkbox)` + margin-right: 12px; + padding-bottom: 0px; +`; + +type Props = { + isSelectAll: boolean; + selectedEntities?: EntityAndType[]; + showCancel?: boolean; + showActions?: boolean; + onChangeSelectAll: (selected: boolean) => void; + onCancel?: () => void; + refetch?: () => void; +}; + +/** + * A header for use when an entity search select experience is active. + * + * This component provides a select all checkbox and a set of actions that can be taken on the selected entities. + */ +export const SearchSelectBar = ({ + isSelectAll, + selectedEntities = [], + showCancel = true, + showActions = true, + onChangeSelectAll, + onCancel, + refetch, +}: Props) => { + const selectedEntityCount = selectedEntities.length; + const onClickCancel = () => { + if (selectedEntityCount > 0) { + Modal.confirm({ + title: `Exit Selection`, + content: `Are you sure you want to exit? ${selectedEntityCount} selection(s) will be cleared.`, + onOk() { + onCancel?.(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + } else { + onCancel?.(); + } + }; + + return ( + <> + + onChangeSelectAll(e.target.checked as boolean)} + /> + + {selectedEntityCount} selected + + + + {showActions && } + {showCancel && ( + + Done + + )} + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectModal.tsx new file mode 100644 index 00000000000000..b7ee4e72433afc --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchSelectModal.tsx @@ -0,0 +1,93 @@ +import { Button, Modal } from 'antd'; +import React, { useState } from 'react'; +import styled from 'styled-components'; +import { EntityType } from '../../../../../../types.generated'; +import ClickOutside from '../../../../../shared/ClickOutside'; +import { EntityAndType } from '../../../types'; +import { SearchSelect } from './SearchSelect'; + +const StyledModal = styled(Modal)` + top: 30px; +`; + +const MODAL_WIDTH_PX = 800; + +const MODAL_BODY_STYLE = { padding: 0, height: '70vh' }; + +type Props = { + fixedEntityTypes?: Array | null; + placeholderText?: string | null; + titleText?: string | null; + continueText?: string | null; + onContinue: (entityUrns: string[]) => void; + onCancel?: () => void; +}; + +/** + * Modal that can be used for searching & selecting a subset of the entities in the Metadata Graph in order to take a specific action. + * + * This component provides easy ways to filter for a specific set of entity types, and provides a set of entity urns + * when the selection is complete. + */ +export const SearchSelectModal = ({ + fixedEntityTypes, + placeholderText, + titleText, + continueText, + onContinue, + onCancel, +}: Props) => { + const [selectedEntities, setSelectedEntities] = useState([]); + + const onCancelSelect = () => { + if (selectedEntities.length > 0) { + Modal.confirm({ + title: `Exit Selection`, + content: `Are you sure you want to exit? ${selectedEntities.length} selection(s) will be cleared.`, + onOk() { + onCancel?.(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + } else { + onCancel?.(); + } + }; + + return ( + + + + + + } + > + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/ActionDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/ActionDropdown.tsx new file mode 100644 index 00000000000000..8158f052a6eb16 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/ActionDropdown.tsx @@ -0,0 +1,74 @@ +import React from 'react'; +import { Button, Dropdown, Menu, Tooltip } from 'antd'; +import { CaretDownOutlined } from '@ant-design/icons'; +import styled from 'styled-components'; +import MenuItem from 'antd/lib/menu/MenuItem'; +import { ANTD_GRAY } from '../../../../constants'; + +const DownArrow = styled(CaretDownOutlined)` + && { + padding-top: 4px; + font-size: 8px; + margin-left: 2px; + margin-top: 2px; + color: ${ANTD_GRAY[7]}; + } +`; + +const StyledMenuItem = styled(MenuItem)` + && { + padding: 0px; + } +`; + +const ActionButton = styled(Button)` + font-weight: normal; +`; + +const DropdownWrapper = styled.div<{ + disabled: boolean; +}>` + cursor: ${(props) => (props.disabled ? 'normal' : 'pointer')}; + color: ${(props) => (props.disabled ? ANTD_GRAY[7] : 'none')}; + display: flex; + margin-left: 12px; + margin-right: 12px; +`; + +export type Action = { + title: React.ReactNode; + onClick: () => void; +}; + +type Props = { + name: string; + actions: Array; + disabled?: boolean; +}; + +export default function ActionDropdown({ name, actions, disabled }: Props) { + return ( + + + {actions.map((action) => ( + + + {action.title} + + + ))} + + } + > + + {name} + + + + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/DeleteDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/DeleteDropdown.tsx new file mode 100644 index 00000000000000..5a13c840373131 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/DeleteDropdown.tsx @@ -0,0 +1,64 @@ +import { message, Modal } from 'antd'; +import React from 'react'; +import { useBatchUpdateSoftDeletedMutation } from '../../../../../../../graphql/mutations.generated'; +import ActionDropdown from './ActionDropdown'; + +type Props = { + urns: Array; + disabled: boolean; + refetch?: () => void; +}; + +// eslint-disable-next-line +export default function DeleteDropdown({ urns, disabled = false, refetch }: Props) { + const [batchUpdateSoftDeletedMutation] = useBatchUpdateSoftDeletedMutation(); + + const batchSoftDelete = () => { + batchUpdateSoftDeletedMutation({ + variables: { + input: { + urns, + deleted: true, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ content: 'Deleted assets!', duration: 2 }); + setTimeout(() => refetch?.(), 3000); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to delete assets: \n ${e.message || ''}`, duration: 3 }); + }); + }; + + return ( + <> + { + Modal.confirm({ + title: `Confirm Delete`, + content: `Are you sure you want to mark these assets as deleted? This will hide the assets + from future DataHub searches. If the assets are re-ingested from an external data platform, they will be restored.`, + onOk() { + batchSoftDelete(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }, + }, + ]} + disabled={disabled} + /> + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/DeprecationDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/DeprecationDropdown.tsx new file mode 100644 index 00000000000000..9e4c22e1748273 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/DeprecationDropdown.tsx @@ -0,0 +1,80 @@ +import { message, Modal } from 'antd'; +import React, { useState } from 'react'; +import { useBatchUpdateDeprecationMutation } from '../../../../../../../graphql/mutations.generated'; +import { UpdateDeprecationModal } from '../../../../EntityDropdown/UpdateDeprecationModal'; +import ActionDropdown from './ActionDropdown'; + +type Props = { + urns: Array; + disabled: boolean; + refetch?: () => void; +}; + +// eslint-disable-next-line +export default function DeprecationDropdown({ urns, disabled = false, refetch }: Props) { + const [isEditModalVisible, setIsEditModalVisible] = useState(false); + const [batchUpdateDeprecationMutation] = useBatchUpdateDeprecationMutation(); + + const batchUndeprecate = () => { + batchUpdateDeprecationMutation({ + variables: { + input: { + resources: [...urns.map((urn) => ({ resourceUrn: urn }))], + deprecated: false, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ content: 'Marked assets as undeprecated!', duration: 2 }); + refetch?.(); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to mark assets as undeprecated: \n ${e.message || ''}`, duration: 3 }); + }); + }; + + return ( + <> + { + setIsEditModalVisible(true); + }, + }, + { + title: 'Mark as undeprecated', + onClick: () => { + Modal.confirm({ + title: `Confirm Mark as undeprecated`, + content: `Are you sure you want to mark these assets as undeprecated?`, + onOk() { + batchUndeprecate(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }, + }, + ]} + disabled={disabled} + /> + {isEditModalVisible && ( + { + setIsEditModalVisible(false); + refetch?.(); + }} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/DomainsDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/DomainsDropdown.tsx new file mode 100644 index 00000000000000..7ce8222aad379f --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/DomainsDropdown.tsx @@ -0,0 +1,79 @@ +import { message, Modal } from 'antd'; +import React, { useState } from 'react'; +import { useBatchSetDomainMutation } from '../../../../../../../graphql/mutations.generated'; +import { SetDomainModal } from '../../../../containers/profile/sidebar/Domain/SetDomainModal'; +import ActionDropdown from './ActionDropdown'; + +type Props = { + urns: Array; + disabled: boolean; + refetch?: () => void; +}; + +// eslint-disable-next-line +export default function DomainsDropdown({ urns, disabled = false, refetch }: Props) { + const [isEditModalVisible, setIsEditModalVisible] = useState(false); + const [batchSetDomainMutation] = useBatchSetDomainMutation(); + + const batchUnsetDomains = () => { + batchSetDomainMutation({ + variables: { + input: { + resources: [...urns.map((urn) => ({ resourceUrn: urn }))], + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ content: 'Removed Domain!', duration: 2 }); + refetch?.(); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to remove assets from Domain: \n ${e.message || ''}`, duration: 3 }); + }); + }; + + return ( + <> + { + setIsEditModalVisible(true); + }, + }, + { + title: 'Unset Domain', + onClick: () => { + Modal.confirm({ + title: `If you continue, Domain will be removed for the selected assets.`, + content: `Are you sure you want to unset Domain for these assets?`, + onOk() { + batchUnsetDomains(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }, + }, + ]} + disabled={disabled} + /> + {isEditModalVisible && ( + { + setIsEditModalVisible(false); + refetch?.(); + }} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/GlossaryTermsDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/GlossaryTermsDropdown.tsx new file mode 100644 index 00000000000000..6c481641214784 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/GlossaryTermsDropdown.tsx @@ -0,0 +1,56 @@ +import React, { useState } from 'react'; +import { EntityType } from '../../../../../../../types.generated'; +import EditTagTermsModal, { OperationType } from '../../../../../../shared/tags/AddTagsTermsModal'; +import ActionDropdown from './ActionDropdown'; + +type Props = { + urns: Array; + disabled: boolean; + refetch?: () => void; +}; + +// eslint-disable-next-line +export default function GlossaryTermsDropdown({ urns, disabled = false, refetch }: Props) { + const [isEditModalVisible, setIsEditModalVisible] = useState(false); + const [operationType, setOperationType] = useState(OperationType.ADD); + + return ( + <> + { + setOperationType(OperationType.ADD); + setIsEditModalVisible(true); + }, + }, + { + title: 'Remove Glossary Terms', + onClick: () => { + setOperationType(OperationType.REMOVE); + setIsEditModalVisible(true); + }, + }, + ]} + disabled={disabled} + /> + {isEditModalVisible && ( + { + setIsEditModalVisible(false); + refetch?.(); + }} + resources={urns.map((urn) => ({ + resourceUrn: urn, + }))} + operationType={operationType} + entityType={EntityType.Dataset} // TODO REMOVE + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/OwnersDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/OwnersDropdown.tsx new file mode 100644 index 00000000000000..5482d287ada321 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/OwnersDropdown.tsx @@ -0,0 +1,51 @@ +import React, { useState } from 'react'; +import { EditOwnersModal, OperationType } from '../../../../containers/profile/sidebar/Ownership/EditOwnersModal'; +import ActionDropdown from './ActionDropdown'; + +type Props = { + urns: Array; + disabled: boolean; + refetch?: () => void; +}; + +// eslint-disable-next-line +export default function OwnersDropdown({ urns, disabled = false, refetch }: Props) { + const [isEditModalVisible, setIsEditModalVisible] = useState(false); + const [operationType, setOperationType] = useState(OperationType.ADD); + + return ( + <> + { + setOperationType(OperationType.ADD); + setIsEditModalVisible(true); + }, + }, + { + title: 'Remove owners', + onClick: () => { + setOperationType(OperationType.REMOVE); + setIsEditModalVisible(true); + }, + }, + ]} + disabled={disabled} + /> + {isEditModalVisible && ( + { + setIsEditModalVisible(false); + refetch?.(); + }} + hideOwnerType={operationType === OperationType.REMOVE} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/action/TagsDropdown.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/action/TagsDropdown.tsx new file mode 100644 index 00000000000000..2c46db8a70213c --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/action/TagsDropdown.tsx @@ -0,0 +1,56 @@ +import React, { useState } from 'react'; +import { EntityType } from '../../../../../../../types.generated'; +import EditTagTermsModal, { OperationType } from '../../../../../../shared/tags/AddTagsTermsModal'; +import ActionDropdown from './ActionDropdown'; + +type Props = { + urns: Array; + disabled: boolean; + refetch?: () => void; +}; + +// eslint-disable-next-line +export default function TagsDropdown({ urns, disabled = false, refetch }: Props) { + const [isEditModalVisible, setIsEditModalVisible] = useState(false); + const [operationType, setOperationType] = useState(OperationType.ADD); + + return ( + <> + { + setOperationType(OperationType.ADD); + setIsEditModalVisible(true); + }, + }, + { + title: 'Remove tags', + onClick: () => { + setOperationType(OperationType.REMOVE); + setIsEditModalVisible(true); + }, + }, + ]} + disabled={disabled} + /> + {isEditModalVisible && ( + { + setIsEditModalVisible(false); + refetch?.(); + }} + resources={urns.map((urn) => ({ + resourceUrn: urn, + }))} + entityType={EntityType.DataFlow} + operationType={operationType} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/downloadAsCsvUtil.ts b/datahub-web-react/src/app/entity/shared/components/styled/search/downloadAsCsvUtil.ts index dc16c9e07a1e11..23d6569de2138a 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/downloadAsCsvUtil.ts +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/downloadAsCsvUtil.ts @@ -78,7 +78,7 @@ export const transformGenericEntityPropertiesToCsvRow = ( // terms properties?.glossaryTerms?.terms?.map((term) => term.term.name).join(',') || '', // domain - properties?.domain?.properties?.name || '', + properties?.domain?.domain?.properties?.name || '', // properties properties?.platform?.properties?.displayName || '', // container diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/navigateToEntitySearchUrl.ts b/datahub-web-react/src/app/entity/shared/components/styled/search/navigateToEntitySearchUrl.ts index 3b719ae61f984a..e0f59c4f7fa2ac 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/navigateToEntitySearchUrl.ts +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/navigateToEntitySearchUrl.ts @@ -5,6 +5,7 @@ import filtersToQueryStringParams from '../../../../../search/utils/filtersToQue export const navigateToEntitySearchUrl = ({ baseUrl, + baseParams, type: newType, query: newQuery, page: newPage = 1, @@ -12,6 +13,7 @@ export const navigateToEntitySearchUrl = ({ history, }: { baseUrl: string; + baseParams: Record; type?: EntityType; query?: string; page?: number; @@ -28,6 +30,7 @@ export const navigateToEntitySearchUrl = ({ ...filtersToQueryStringParams(constructedFilters), query: newQuery, page: newPage, + ...baseParams, }, { arrayFormat: 'comma' }, ); diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/types.ts b/datahub-web-react/src/app/entity/shared/components/styled/search/types.ts index 5ccfd66b7a8f5f..9d8ed6e97e7622 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/types.ts +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/types.ts @@ -35,3 +35,15 @@ export type SearchResultsInterface = { /** Candidate facet aggregations used for search filtering */ facets?: Maybe>; }; + +/** + * Supported Action Groups for search-select feature. + */ +export enum SelectActionGroups { + CHANGE_OWNERS, + CHANGE_TAGS, + CHANGE_GLOSSARY_TERMS, + CHANGE_DOMAINS, + CHANGE_DEPRECATION, + DELETE, +} diff --git a/datahub-web-react/src/app/entity/shared/constants.ts b/datahub-web-react/src/app/entity/shared/constants.ts index 51d785c12fd4ae..a1412e4133437a 100644 --- a/datahub-web-react/src/app/entity/shared/constants.ts +++ b/datahub-web-react/src/app/entity/shared/constants.ts @@ -47,4 +47,12 @@ export const EMPTY_MESSAGES = { title: 'No domain set', description: 'Group related entities based on your organizational structure using by adding them to a Domain.', }, + contains: { + title: 'Contains no Terms', + description: 'Terms can contain other terms to represent an "Has A" style relationship.', + }, + inherits: { + title: 'Does not inherit from any terms', + description: 'Terms can inherit from other terms to represent an "Is A" style relationship.', + }, }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx index 036c800850e523..f8c69fc7ae7511 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx @@ -6,7 +6,13 @@ import { useHistory } from 'react-router'; import { EntityType, Exact } from '../../../../../types.generated'; import { Message } from '../../../../shared/Message'; import { getDataForEntityType, getEntityPath, useRoutedTab } from './utils'; -import { EntitySidebarSection, EntityTab, GenericEntityProperties, GenericEntityUpdate } from '../../types'; +import { + EntitySidebarSection, + EntitySubHeaderSection, + EntityTab, + GenericEntityProperties, + GenericEntityUpdate, +} from '../../types'; import { EntityProfileNavBar } from './nav/EntityProfileNavBar'; import { ANTD_GRAY } from '../../constants'; import { EntityHeader } from './header/EntityHeader'; @@ -24,6 +30,8 @@ import { EntityMenuItems } from '../../EntityDropdown/EntityDropdown'; import GlossaryBrowser from '../../../../glossary/GlossaryBrowser/GlossaryBrowser'; import GlossarySearch from '../../../../glossary/GlossarySearch'; import { BrowserWrapper, MAX_BROWSER_WIDTH, MIN_BROWSWER_WIDTH } from '../../../../glossary/BusinessGlossaryPage'; +import { combineEntityDataWithSiblings, useIsSeparateSiblingsMode } from '../../siblingUtils'; +import { EntityActionItem } from '../../entity/EntityActions'; type Props = { urn: string; @@ -48,7 +56,9 @@ type Props = { tabs: EntityTab[]; sidebarSections: EntitySidebarSection[]; customNavBar?: React.ReactNode; + subHeader?: EntitySubHeaderSection; headerDropdownItems?: Set; + headerActionItems?: Set; displayGlossaryBrowser?: boolean; isNameEditable?: boolean; }; @@ -63,7 +73,6 @@ const ContentContainer = styled.div` const HeaderAndTabs = styled.div` flex-grow: 1; min-width: 640px; - height: 100%; `; const HeaderAndTabsFlex = styled.div` @@ -90,7 +99,8 @@ const HeaderAndTabsFlex = styled.div` const Sidebar = styled.div<{ $width: number }>` max-height: 100%; overflow: auto; - flex-basis: ${(props) => props.$width}px; + width: ${(props) => props.$width}px; + min-width: ${(props) => props.$width}px; padding-left: 20px; padding-right: 20px; `; @@ -117,7 +127,6 @@ const defaultSidebarSection = { visible: (_, _1) => true, }; -const INITIAL_SIDEBAR_WIDTH = 400; const MAX_SIDEBAR_WIDTH = 800; const MIN_SIDEBAR_WIDTH = 200; @@ -134,10 +143,13 @@ export const EntityProfile = ({ sidebarSections, customNavBar, headerDropdownItems, + headerActionItems, displayGlossaryBrowser, isNameEditable, + subHeader, }: Props): JSX.Element => { const isLineageMode = useIsLineageMode(); + const isHideSiblingMode = useIsSeparateSiblingsMode(); const entityRegistry = useEntityRegistry(); const history = useHistory(); const isCompact = React.useContext(CompactContext); @@ -147,7 +159,7 @@ export const EntityProfile = ({ display: { ...defaultSidebarSection, ...sidebarSection.display }, })); - const [sidebarWidth, setSidebarWidth] = useState(INITIAL_SIDEBAR_WIDTH); + const [sidebarWidth, setSidebarWidth] = useState(window.innerWidth * 0.25); const [browserWidth, setBrowserWith] = useState(window.innerWidth * 0.2); const [shouldUpdateBrowser, setShouldUpdateBrowser] = useState(false); @@ -172,15 +184,26 @@ export const EntityProfile = ({ entityUrn: urn, section: tabName.toLowerCase(), }); - history[method](getEntityPath(entityType, urn, entityRegistry, false, tabName, tabParams)); + history[method]( + getEntityPath(entityType, urn, entityRegistry, false, isHideSiblingMode, tabName, tabParams), + ); }, - [history, entityType, urn, entityRegistry], + [history, entityType, urn, entityRegistry, isHideSiblingMode], ); - const { loading, error, data, refetch } = useEntityQuery({ + const { + loading, + error, + data: dataNotCombinedWithSiblings, + refetch, + } = useEntityQuery({ variables: { urn }, }); + const dataPossiblyCombinedWithSiblings = isHideSiblingMode + ? dataNotCombinedWithSiblings + : combineEntityDataWithSiblings(dataNotCombinedWithSiblings); + const maybeUpdateEntity = useUpdateQuery?.({ onCompleted: () => refetch(), }); @@ -190,7 +213,15 @@ export const EntityProfile = ({ } const entityData = - (data && getDataForEntityType({ data: data[Object.keys(data)[0]], entityType, getOverrideProperties })) || null; + (dataPossiblyCombinedWithSiblings && + Object.keys(dataPossiblyCombinedWithSiblings).length > 0 && + getDataForEntityType({ + data: dataPossiblyCombinedWithSiblings[Object.keys(dataPossiblyCombinedWithSiblings)[0]], + entityType, + getOverrideProperties, + isHideSiblingMode, + })) || + null; const lineage = entityData ? entityRegistry.getLineageVizConfig(entityType, entityData) : undefined; @@ -219,7 +250,8 @@ export const EntityProfile = ({ urn, entityType, entityData, - baseEntity: data, + baseEntity: dataPossiblyCombinedWithSiblings, + dataNotCombinedWithSiblings, updateEntity, routeToTab, refetch, @@ -233,7 +265,11 @@ export const EntityProfile = ({ )} {!loading && ( <> - + @@ -253,7 +289,8 @@ export const EntityProfile = ({ urn, entityType, entityData, - baseEntity: data, + baseEntity: dataPossiblyCombinedWithSiblings, + dataNotCombinedWithSiblings, updateEntity, routeToTab, refetch, @@ -300,7 +337,9 @@ export const EntityProfile = ({
{ + it('should return true for Terms if manageGlossaries privilege is true', () => { + const canEditName = getCanEditName(EntityType.GlossaryTerm, platformPrivileges); + + expect(canEditName).toBe(true); + }); + + it('should return false for Terms if manageGlossaries privilege is false', () => { + const privilegesWithoutGlossaries = { ...platformPrivileges, manageGlossaries: false }; + const canEditName = getCanEditName(EntityType.GlossaryTerm, privilegesWithoutGlossaries); + + expect(canEditName).toBe(false); + }); + + it('should return true for Nodes if manageGlossaries privilege is true', () => { + const canEditName = getCanEditName(EntityType.GlossaryNode, platformPrivileges); + + expect(canEditName).toBe(true); + }); + + it('should return false for Nodes if manageGlossaries privilege is false', () => { + const privilegesWithoutGlossaries = { ...platformPrivileges, manageGlossaries: false }; + const canEditName = getCanEditName(EntityType.GlossaryNode, privilegesWithoutGlossaries); + + expect(canEditName).toBe(false); + }); + + it('should return true for Domains if manageDomains privilege is true', () => { + const canEditName = getCanEditName(EntityType.Domain, platformPrivileges); + + expect(canEditName).toBe(true); + }); + + it('should return false for Domains if manageDomains privilege is false', () => { + const privilegesWithoutDomains = { ...platformPrivileges, manageDomains: false }; + const canEditName = getCanEditName(EntityType.Domain, privilegesWithoutDomains); + + expect(canEditName).toBe(false); + }); + + it('should return false for an unsupported entity', () => { + const canEditName = getCanEditName(EntityType.Chart, platformPrivileges); + + expect(canEditName).toBe(false); + }); +}); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx index 2d2c7cf7b2f8aa..81f83e6c55ab3f 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx @@ -1,14 +1,11 @@ import React, { useState } from 'react'; -import { InfoCircleOutlined, RightOutlined } from '@ant-design/icons'; -import { Typography, Button, Tooltip, Popover } from 'antd'; +import { ArrowRightOutlined } from '@ant-design/icons'; +import { Button } from 'antd'; import styled from 'styled-components/macro'; -import moment from 'moment'; import { capitalizeFirstLetterOnly } from '../../../../../shared/textUtil'; -import { ANTD_GRAY } from '../../../constants'; -import { useEntityData } from '../../../EntityContext'; +import { useEntityData, useRefetch } from '../../../EntityContext'; import analytics, { EventType, EntityActionType } from '../../../../../analytics'; import { EntityHealthStatus } from './EntityHealthStatus'; -import { getLocaleTimezone } from '../../../../../shared/time/timeUtils'; import EntityDropdown, { EntityMenuItems } from '../../../EntityDropdown/EntityDropdown'; import PlatformContent from './PlatformContent'; import { getPlatformName } from '../../../utils'; @@ -17,6 +14,10 @@ import { EntityType, PlatformPrivileges } from '../../../../../../types.generate import EntityCount from './EntityCount'; import EntityName from './EntityName'; import CopyUrn from '../../../../../shared/CopyUrn'; +import { DeprecationPill } from '../../../components/styled/DeprecationPill'; +import CompactContext from '../../../../../shared/CompactContext'; +import { EntitySubHeaderSection } from '../../../types'; +import EntityActions, { EntityActionItem } from '../../../entity/EntityActions'; const TitleWrapper = styled.div` display: flex; @@ -45,38 +46,6 @@ const MainHeaderContent = styled.div` } `; -const DeprecatedContainer = styled.div` - width: 110px; - height: 18px; - border: 1px solid #ef5b5b; - border-radius: 15px; - display: flex; - justify-content: center; - align-items: center; - color: #ef5b5b; - margin-left: 15px; - padding-top: 12px; - padding-bottom: 12px; -`; - -const DeprecatedText = styled.div` - color: #ef5b5b; - margin-left: 5px; -`; - -const LastEvaluatedAtLabel = styled.div` - padding: 0; - margin: 0; - display: flex; - align-items: center; - color: ${ANTD_GRAY[7]}; -`; - -const Divider = styled.div` - border-top: 1px solid #f0f0f0; - padding-top: 5px; -`; - const SideHeaderContent = styled.div` display: flex; flex-direction: column; @@ -88,11 +57,25 @@ const TopButtonsWrapper = styled.div` margin-bottom: 8px; `; -function getCanEditName(entityType: EntityType, privileges?: PlatformPrivileges) { +const ExternalUrlContainer = styled.span` + font-size: 14px; +`; + +const ExternalUrlButton = styled(Button)` + > :hover { + text-decoration: underline; + } + padding-left: 12px; + padding-right: 12px; +`; + +export function getCanEditName(entityType: EntityType, privileges?: PlatformPrivileges) { switch (entityType) { case EntityType.GlossaryTerm: case EntityType.GlossaryNode: return privileges?.manageGlossaries; + case EntityType.Domain: + return privileges?.manageDomains; default: return false; } @@ -101,18 +84,27 @@ function getCanEditName(entityType: EntityType, privileges?: PlatformPrivileges) type Props = { refreshBrowser?: () => void; headerDropdownItems?: Set; + headerActionItems?: Set; isNameEditable?: boolean; + subHeader?: EntitySubHeaderSection; }; -export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditable }: Props) => { +export const EntityHeader = ({ + refreshBrowser, + headerDropdownItems, + headerActionItems, + isNameEditable, + subHeader, +}: Props) => { const { urn, entityType, entityData } = useEntityData(); + const refetch = useRefetch(); const me = useGetAuthenticatedUser(); const [copiedUrn, setCopiedUrn] = useState(false); const basePlatformName = getPlatformName(entityData); const platformName = capitalizeFirstLetterOnly(basePlatformName); const externalUrl = entityData?.externalUrl || undefined; const entityCount = entityData?.entityCount; - const hasExternalUrl = !!externalUrl; + const isCompact = React.useContext(CompactContext); const sendAnalytics = () => { analytics.event({ @@ -123,89 +115,61 @@ export const EntityHeader = ({ refreshBrowser, headerDropdownItems, isNameEditab }); }; - /** - * Deprecation Decommission Timestamp - */ - const localeTimezone = getLocaleTimezone(); - const decommissionTimeLocal = - (entityData?.deprecation?.decommissionTime && - `Scheduled to be decommissioned on ${moment - .unix(entityData?.deprecation?.decommissionTime) - .format('DD/MMM/YYYY')} at ${moment - .unix(entityData?.deprecation?.decommissionTime) - .format('HH:mm:ss')} (${localeTimezone})`) || - undefined; - const decommissionTimeGMT = - entityData?.deprecation?.decommissionTime && - moment.unix(entityData?.deprecation?.decommissionTime).utc().format('dddd, DD/MMM/YYYY HH:mm:ss z'); - - const hasDetails = entityData?.deprecation?.note !== '' || entityData?.deprecation?.decommissionTime !== null; - const isDividerNeeded = entityData?.deprecation?.note !== '' && entityData?.deprecation?.decommissionTime !== null; const canEditName = isNameEditable && getCanEditName(entityType, me?.platformPrivileges as PlatformPrivileges); return ( - - - - - - {entityData?.deprecation?.deprecated && ( - - {entityData?.deprecation?.note !== '' && ( - {entityData?.deprecation?.note} - )} - {isDividerNeeded && } - {entityData?.deprecation?.decommissionTime !== null && ( - - - {decommissionTimeLocal} - - - )} - - ) : ( - 'No additional details' - ) - } - > - - - Deprecated - - - )} - {entityData?.health && ( - - )} - - - - - - setCopiedUrn(true)} /> - {headerDropdownItems && ( - - )} - - {hasExternalUrl && ( - - )} - - + <> + + + + + + {entityData?.deprecation?.deprecated && ( + + )} + {entityData?.health?.map((health) => ( + + ))} + + + + + + {externalUrl && ( + + + View in {platformName} + + + )} + {headerActionItems && ( + + )} + setCopiedUrn(true)} /> + {headerDropdownItems && ( + + )} + + + + {subHeader && } + ); }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx index 09801f92aa1908..4d16fc9ac0bff2 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHealthStatus.tsx @@ -1,18 +1,27 @@ import React from 'react'; import { Tooltip } from 'antd'; +import styled from 'styled-components'; import { getHealthIcon } from '../../../../../shared/health/healthUtils'; -import { HealthStatus } from '../../../../../../types.generated'; +import { HealthStatus, HealthStatusType } from '../../../../../../types.generated'; + +const StatusContainer = styled.div` + display: flex; + justify-content: center; + align-items: center; + margin-left: 8px; +`; type Props = { + type: HealthStatusType; status: HealthStatus; message?: string | undefined; }; -export const EntityHealthStatus = ({ status, message }: Props) => { - const icon = getHealthIcon(status, 18); +export const EntityHealthStatus = ({ type, status, message }: Props) => { + const icon = getHealthIcon(type, status, 18); return ( -
+ {icon} -
+ ); }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx index 90c543dc3641bf..b5004ed96dbf01 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx @@ -6,6 +6,8 @@ import { useEntityRegistry } from '../../../../../useEntityRegistry'; import { useEntityData, useRefetch } from '../../../EntityContext'; const EntityTitle = styled(Typography.Title)` + margin-right: 10px; + &&& { margin-bottom: 0; word-break: break-all; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx index 034c8fc292b3ab..6c45054898083e 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx @@ -45,6 +45,10 @@ function PlatformContentContainer() { platform.properties?.displayName || platform.name, + )} + platformLogoUrls={entityData?.siblingPlatforms?.map((platform) => platform.properties?.logoUrl)} entityLogoComponent={entityLogoComponent} instanceId={instanceId} typeIcon={typeIcon} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx index 14329a1d437268..24483d0fd9d716 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx @@ -6,6 +6,7 @@ import { Maybe } from 'graphql/jsutils/Maybe'; import { Container } from '../../../../../../../types.generated'; import { ANTD_GRAY } from '../../../../constants'; import ContainerLink from './ContainerLink'; +import { capitalizeFirstLetterOnly } from '../../../../../../shared/textUtil'; const LogoIcon = styled.span` display: flex; @@ -87,6 +88,8 @@ export function getParentContainerNames(containers?: Maybe[] | null) interface Props { platformName?: string; platformLogoUrl?: Maybe; + platformNames?: Maybe[]; + platformLogoUrls?: Maybe[]; entityLogoComponent?: JSX.Element; instanceId?: string; typeIcon?: JSX.Element; @@ -100,6 +103,8 @@ function PlatformContentView(props: Props) { const { platformName, platformLogoUrl, + platformNames, + platformLogoUrls, entityLogoComponent, instanceId, typeIcon, @@ -115,16 +120,24 @@ function PlatformContentView(props: Props) { return ( {typeIcon && {typeIcon}} - {entityType} + {capitalizeFirstLetterOnly(entityType)} {(!!platformName || !!instanceId || !!parentContainers?.length) && } {platformName && ( - {(!!platformLogoUrl && ) || - entityLogoComponent} + {!platformLogoUrl && !platformLogoUrls && entityLogoComponent} + {!!platformLogoUrl && !platformLogoUrls && ( + + )} + {!!platformLogoUrls && + platformLogoUrls.slice(0, 2).map((platformLogoUrlsEntry) => ( + <> + + + ))} )} - {platformName} + {platformNames ? platformNames.join(' & ') : platformName} {(directParentContainer || instanceId) && } {instanceId && ( diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx index 6cf9fb514c9b51..d1492b02452f07 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/nav/EntityProfileNavBar.tsx @@ -27,8 +27,8 @@ export const EntityProfileNavBar = ({ urn, entityType }: Props) => { breadcrumbLinksEnabled={isBrowsable} type={entityType} path={browseData?.browsePaths?.[0]?.path || []} - upstreams={lineage?.upstreamChildren?.length || 0} - downstreams={lineage?.downstreamChildren?.length || 0} + upstreams={lineage?.numUpstreamChildren || 0} + downstreams={lineage?.numDownstreamChildren || 0} /> ); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx index d0db0fe5be4406..eafbc91b853445 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx @@ -1,82 +1,88 @@ -import { Button, Form, message, Modal, Select, Tag } from 'antd'; import React, { useRef, useState } from 'react'; +import { Button, Form, message, Modal, Select, Tag } from 'antd'; import styled from 'styled-components'; -import { Link } from 'react-router-dom'; + import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/search.generated'; -import { EntityType, SearchResult } from '../../../../../../../types.generated'; -import { useSetDomainMutation } from '../../../../../../../graphql/mutations.generated'; +import { Entity, EntityType } from '../../../../../../../types.generated'; +import { useBatchSetDomainMutation } from '../../../../../../../graphql/mutations.generated'; import { useEntityRegistry } from '../../../../../../useEntityRegistry'; -import { useEntityData } from '../../../../EntityContext'; import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener'; +import { useGetRecommendations } from '../../../../../../shared/recommendation'; +import { DomainLabel } from '../../../../../../shared/DomainLabel'; type Props = { - visible: boolean; - onClose: () => void; + urns: string[]; + onCloseModal: () => void; refetch?: () => Promise; }; -const SearchResultContainer = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - padding: 12px; -`; - -const SearchResultContent = styled.div` - display: flex; - justify-content: start; - align-items: center; -`; - -const SearchResultDisplayName = styled.div` - margin-left: 12px; -`; - type SelectedDomain = { displayName: string; type: EntityType; urn: string; }; -export const SetDomainModal = ({ visible, onClose, refetch }: Props) => { +const StyleTag = styled(Tag)` + padding: 0px 7px; + margin-right: 3px; + display: flex; + justify-content: start; + align-items: center; +`; + +export const SetDomainModal = ({ urns, onCloseModal, refetch }: Props) => { const entityRegistry = useEntityRegistry(); - const { urn } = useEntityData(); + const [inputValue, setInputValue] = useState(''); const [selectedDomain, setSelectedDomain] = useState(undefined); const [domainSearch, { data: domainSearchData }] = useGetSearchResultsLazyQuery(); - const domainSearchResults = domainSearchData?.search?.searchResults || []; - const [setDomainMutation] = useSetDomainMutation(); - + const domainSearchResults = + domainSearchData?.search?.searchResults?.map((searchResult) => searchResult.entity) || []; + const [batchSetDomainMutation] = useBatchSetDomainMutation(); + const [recommendedData] = useGetRecommendations([EntityType.Domain]); const inputEl = useRef(null); - const onOk = async () => { - if (!selectedDomain) { - return; - } - try { - await setDomainMutation({ + const onModalClose = () => { + setInputValue(''); + setSelectedDomain(undefined); + onCloseModal(); + }; + + const handleSearch = (text: string) => { + if (text.length > 2) { + domainSearch({ variables: { - entityUrn: urn, - domainUrn: selectedDomain.urn, + input: { + type: EntityType.Domain, + query: text, + start: 0, + count: 5, + }, }, }); - message.success({ content: 'Updated Domain!', duration: 2 }); - } catch (e: unknown) { - message.destroy(); - if (e instanceof Error) { - message.error({ content: `Failed to set Domain: \n ${e.message || ''}`, duration: 3 }); - } } - setSelectedDomain(undefined); - refetch?.(); - onClose(); }; + // Renders a search result in the select dropdown. + const renderSearchResult = (entity: Entity) => { + const displayName = entityRegistry.getDisplayName(entity.type, entity); + return ( + + + + ); + }; + + const domainResult = !inputValue || inputValue.length === 0 ? recommendedData : domainSearchResults; + + const domainSearchOptions = domainResult?.map((result) => { + return renderSearchResult(result); + }); + const onSelectDomain = (newUrn: string) => { if (inputEl && inputEl.current) { (inputEl.current as any).blur(); } - const filteredDomains = - domainSearchResults?.filter((result) => result.entity.urn === newUrn).map((result) => result.entity) || []; + const filteredDomains = domainResult?.filter((entity) => entity.urn === newUrn).map((entity) => entity) || []; if (filteredDomains.length) { const domain = filteredDomains[0]; setSelectedDomain({ @@ -87,56 +93,70 @@ export const SetDomainModal = ({ visible, onClose, refetch }: Props) => { } }; - const handleSearch = (text: string) => { - if (text.length > 2) { - domainSearch({ - variables: { - input: { - type: EntityType.Domain, - query: text, - start: 0, - count: 5, - }, + const onDeselectDomain = () => { + setInputValue(''); + setSelectedDomain(undefined); + }; + + const onOk = async () => { + if (!selectedDomain) { + return; + } + batchSetDomainMutation({ + variables: { + input: { + resources: [...urns.map((urn) => ({ resourceUrn: urn }))], + domainUrn: selectedDomain.urn, }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ content: 'Updated Domain!', duration: 2 }); + refetch?.(); + onModalClose(); + setSelectedDomain(undefined); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to add assets to Domain: \n ${e.message || ''}`, duration: 3 }); }); - } }; + const selectValue = (selectedDomain && [selectedDomain?.displayName]) || undefined; + // Handle the Enter press useEnterKeyListener({ querySelectorToExecuteClick: '#setDomainButton', }); - const renderSearchResult = (result: SearchResult) => { - const displayName = entityRegistry.getDisplayName(result.entity.type, result.entity); + const tagRender = (props) => { + // eslint-disable-next-line react/prop-types + const { label, closable, onClose } = props; + const onPreventMouseDown = (event) => { + event.preventDefault(); + event.stopPropagation(); + }; return ( - - - -
{displayName}
- - - `/${entityRegistry.getPathName(result.entity.type)}/${result.entity.urn}`} - > - View - {' '} - + + {label} + ); }; - const selectValue = (selectedDomain && [selectedDomain?.displayName]) || []; + function handleBlur() { + setInputValue(''); + } return ( - } > -
+ Owner}> Find a user or group onSelectOwner(asset)} onDeselect={(asset: any) => onDeselectOwner(asset)} - onSearch={handleActorSearch} + onSearch={(value: string) => { + // eslint-disable-next-line react/prop-types + handleActorSearch(value.trim()); + // eslint-disable-next-line react/prop-types + setInputValue(value.trim()); + }} tagRender={tagRender} + onBlur={handleBlur} + value={selectedOwners} > - {combinedSearchResults?.map((result) => ( - - {renderSearchResult(result)} - - ))} + {ownerSearchOptions} diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/SidebarOwnerSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/SidebarOwnerSection.tsx index 6d431d1680c07f..3ee949732eddfc 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/SidebarOwnerSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/SidebarOwnerSection.tsx @@ -3,12 +3,14 @@ import React, { useState } from 'react'; import { PlusOutlined } from '@ant-design/icons'; import { ExpandedOwner } from '../../../../components/styled/ExpandedOwner'; import { EMPTY_MESSAGES } from '../../../../constants'; -import { useEntityData, useRefetch } from '../../../../EntityContext'; +import { useEntityData, useMutationUrn, useRefetch } from '../../../../EntityContext'; import { SidebarHeader } from '../SidebarHeader'; -import { AddOwnersModal } from './AddOwnersModal'; +import { EditOwnersModal } from './EditOwnersModal'; export const SidebarOwnerSection = ({ properties }: { properties?: any }) => { - const { urn, entityType, entityData } = useEntityData(); + const { entityType, entityData } = useEntityData(); + const mutationUrn = useMutationUrn(); + const refetch = useRefetch(); const [showAddModal, setShowAddModal] = useState(false); const ownersEmpty = !entityData?.ownership?.owners?.length; @@ -18,7 +20,12 @@ export const SidebarOwnerSection = ({ properties }: { properties?: any }) => {
{entityData?.ownership?.owners?.map((owner) => ( - + ))} {ownersEmpty && ( @@ -30,17 +37,18 @@ export const SidebarOwnerSection = ({ properties }: { properties?: any }) => { Add Owners
- { - setShowAddModal(false); - }} - /> + {showAddModal && ( + { + setShowAddModal(false); + }} + /> + )} ); }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarAboutSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarAboutSection.tsx index 61bae85408c8f2..6391f3b9d9d7b6 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarAboutSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarAboutSection.tsx @@ -14,7 +14,7 @@ const DescriptionTypography = styled(Typography.Paragraph)` `; const SidebarLinkList = styled.div` - margin-left: -15px; + margin: 0 0 10px -15px; min-width: 0; `; @@ -32,10 +32,14 @@ const LinkButton = styled(Button)` overflow: hidden; white-space: nowrap; text-overflow: ellipsis; - line-height: 1; } `; +const SourceButton = styled(LinkButton)` + padding: 0; + margin-top: -5px; +`; + interface Props { hideLinksButton?: boolean; } @@ -49,6 +53,9 @@ export const SidebarAboutSection = ({ properties }: { properties?: Props }) => { const description = entityData?.editableProperties?.description || entityData?.properties?.description; const links = entityData?.institutionalMemory?.elements || []; + const sourceUrl = entityData?.properties?.sourceUrl; + const sourceRef = entityData?.properties?.sourceRef; + const isUntouched = !description && !(links?.length > 0); return ( @@ -117,6 +124,21 @@ export const SidebarAboutSection = ({ properties }: { properties?: Props }) => { )} )} + {sourceRef && ( + <> + + + {sourceUrl ? ( + + + {sourceRef} + + ) : ( + {sourceRef} + )} + + + )} ); }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx new file mode 100644 index 00000000000000..4b0089e6b9214a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx @@ -0,0 +1,53 @@ +import React from 'react'; +import styled from 'styled-components'; + +import { useDataNotCombinedWithSiblings, useEntityData } from '../../../EntityContext'; +import { SidebarHeader } from './SidebarHeader'; +import { CompactEntityNameList } from '../../../../../recommendations/renderer/component/CompactEntityNameList'; +import { Entity } from '../../../../../../types.generated'; +import { SEPARATE_SIBLINGS_URL_PARAM, stripSiblingsFromEntity, useIsSeparateSiblingsMode } from '../../../siblingUtils'; +import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; + +const EntityListContainer = styled.div` + margin-left: -8px; +`; + +export const SidebarSiblingsSection = () => { + const { entityData } = useEntityData(); + const dataNotCombinedWithSiblings = useDataNotCombinedWithSiblings(); + + const isHideSiblingMode = useIsSeparateSiblingsMode(); + + if (!entityData) { + return <>; + } + + if (isHideSiblingMode) { + return ( +
+ + + + +
+ ); + } + + const siblingEntities = entityData?.siblings?.siblings || []; + const entityDataWithoutSiblings = stripSiblingsFromEntity(dataNotCombinedWithSiblings.dataset); + + const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Entity[]; + + return ( +
+ + + + +
+ ); +}; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx index 43f551c9a1991b..dd7f0c440f164e 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx @@ -3,7 +3,7 @@ import styled from 'styled-components'; import TagTermGroup from '../../../../../shared/tags/TagTermGroup'; import { SidebarHeader } from './SidebarHeader'; -import { useEntityData, useRefetch } from '../../../EntityContext'; +import { useEntityData, useMutationUrn, useRefetch } from '../../../EntityContext'; const TermSection = styled.div` margin-top: 20px; @@ -13,7 +13,10 @@ export const SidebarTagsSection = ({ properties }: { properties?: any }) => { const canAddTag = properties?.hasTags; const canAddTerm = properties?.hasTerms; - const { urn, entityType, entityData } = useEntityData(); + const mutationUrn = useMutationUrn(); + + const { entityType, entityData } = useEntityData(); + const refetch = useRefetch(); return ( @@ -24,7 +27,7 @@ export const SidebarTagsSection = ({ properties }: { properties?: any }) => { canAddTag={canAddTag} canRemove showEmptyMessage - entityUrn={urn} + entityUrn={mutationUrn} entityType={entityType} refetch={refetch} /> @@ -35,7 +38,7 @@ export const SidebarTagsSection = ({ properties }: { properties?: any }) => { canAddTerm={canAddTerm} canRemove showEmptyMessage - entityUrn={urn} + entityUrn={mutationUrn} entityType={entityType} refetch={refetch} /> diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts b/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts index 14869e2921aa2f..d467a944e5d71d 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts +++ b/datahub-web-react/src/app/entity/shared/containers/profile/utils.ts @@ -5,14 +5,17 @@ import useIsLineageMode from '../../../../lineage/utils/useIsLineageMode'; import { useEntityRegistry } from '../../../../useEntityRegistry'; import EntityRegistry from '../../../EntityRegistry'; import { EntityTab, GenericEntityProperties } from '../../types'; +import { useIsSeparateSiblingsMode, SEPARATE_SIBLINGS_URL_PARAM } from '../../siblingUtils'; export function getDataForEntityType({ data: entityData, getOverrideProperties, + isHideSiblingMode, }: { data: T; - entityType: EntityType; + entityType?: EntityType; getOverrideProperties: (T) => GenericEntityProperties; + isHideSiblingMode?: boolean; }): GenericEntityProperties | null { if (!entityData) { return null; @@ -33,6 +36,22 @@ export function getDataForEntityType({ globalTags: anyEntityData.tags, }; } + + if (anyEntityData?.siblings?.siblings?.length > 0 && !isHideSiblingMode) { + const genericSiblingProperties: GenericEntityProperties[] = anyEntityData?.siblings?.siblings?.map((sibling) => + getDataForEntityType({ data: sibling, getOverrideProperties: () => ({}) }), + ); + + const allPlatforms = anyEntityData.siblings.isPrimary + ? [anyEntityData.platform, genericSiblingProperties?.[0]?.platform] + : [genericSiblingProperties?.[0]?.platform, anyEntityData.platform]; + + modifiedEntityData = { + ...modifiedEntityData, + siblingPlatforms: allPlatforms, + }; + } + return { ...modifiedEntityData, ...getOverrideProperties(entityData), @@ -44,6 +63,7 @@ export function getEntityPath( urn: string, entityRegistry: EntityRegistry, isLineageMode: boolean, + isHideSiblingMode: boolean, tabName?: string, tabParams?: Record, ) { @@ -52,16 +72,16 @@ export function getEntityPath( if (!tabName) { return `${entityRegistry.getEntityUrl(entityType, urn)}?is_lineage_mode=${isLineageMode}${tabParamsString}`; } - return `${entityRegistry.getEntityUrl( - entityType, - urn, - )}/${tabName}?is_lineage_mode=${isLineageMode}${tabParamsString}`; + return `${entityRegistry.getEntityUrl(entityType, urn)}/${tabName}?is_lineage_mode=${isLineageMode}${ + isHideSiblingMode ? `&${SEPARATE_SIBLINGS_URL_PARAM}=${isHideSiblingMode}` : '' + }${tabParamsString}`; } export function useEntityPath(entityType: EntityType, urn: string, tabName?: string, tabParams?: Record) { const isLineageMode = useIsLineageMode(); + const isHideSiblingMode = useIsSeparateSiblingsMode(); const entityRegistry = useEntityRegistry(); - return getEntityPath(entityType, urn, entityRegistry, isLineageMode, tabName, tabParams); + return getEntityPath(entityType, urn, entityRegistry, isLineageMode, isHideSiblingMode, tabName, tabParams); } export function useRoutedTab(tabs: EntityTab[]): EntityTab | undefined { @@ -77,3 +97,13 @@ export function formatDateString(time: number) { const date = new Date(time); return date.toLocaleDateString('en-US'); } + +export function useEntityQueryParams() { + const isHideSiblingMode = useIsSeparateSiblingsMode(); + const response = {}; + if (isHideSiblingMode) { + response[SEPARATE_SIBLINGS_URL_PARAM] = true; + } + + return response; +} diff --git a/datahub-web-react/src/app/entity/shared/entity/EntityActions.tsx b/datahub-web-react/src/app/entity/shared/entity/EntityActions.tsx new file mode 100644 index 00000000000000..623a8f9b3fc102 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/entity/EntityActions.tsx @@ -0,0 +1,131 @@ +import React, { useState } from 'react'; +import { Button, message } from 'antd'; +import { LinkOutlined } from '@ant-design/icons'; +import { SearchSelectModal } from '../components/styled/search/SearchSelectModal'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { EntityCapabilityType } from '../../Entity'; +import { useBatchAddTermsMutation, useBatchSetDomainMutation } from '../../../../graphql/mutations.generated'; + +export enum EntityActionItem { + /** + * Batch add a Glossary Term to a set of assets + */ + BATCH_ADD_GLOSSARY_TERM, + /** + * Batch add a Domain to a set of assets + */ + BATCH_ADD_DOMAIN, +} + +interface Props { + urn: string; + actionItems: Set; + refetchForEntity?: () => void; +} + +function EntityActions(props: Props) { + // eslint ignore react/no-unused-prop-types + const entityRegistry = useEntityRegistry(); + const { urn, actionItems, refetchForEntity } = props; + const [isBatchAddGlossaryTermModalVisible, setIsBatchAddGlossaryTermModalVisible] = useState(false); + const [isBatchSetDomainModalVisible, setIsBatchSetDomainModalVisible] = useState(false); + const [batchAddTermsMutation] = useBatchAddTermsMutation(); + const [batchSetDomainMutation] = useBatchSetDomainMutation(); + + // eslint-disable-next-line + const batchAddGlossaryTerms = (entityUrns: Array) => { + batchAddTermsMutation({ + variables: { + input: { + termUrns: [urn], + resources: entityUrns.map((entityUrn) => ({ + resourceUrn: entityUrn, + })), + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + refetchForEntity?.(); + setIsBatchAddGlossaryTermModalVisible(false); + message.success({ + content: `Added Glossary Term to entities!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to add glossary term: \n ${e.message || ''}`, duration: 3 }); + }); + }; + + // eslint-disable-next-line + const batchSetDomain = (entityUrns: Array) => { + batchSetDomainMutation({ + variables: { + input: { + domainUrn: urn, + resources: entityUrns.map((entityUrn) => ({ + resourceUrn: entityUrn, + })), + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + refetchForEntity?.(); + setIsBatchSetDomainModalVisible(false); + message.success({ + content: `Added assets to Domain!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to add assets to Domain: \n ${e.message || ''}`, duration: 3 }); + }); + }; + + return ( + <> +
+ {actionItems.has(EntityActionItem.BATCH_ADD_GLOSSARY_TERM) && ( + + )} + {actionItems.has(EntityActionItem.BATCH_ADD_DOMAIN) && ( + + )} +
+ {isBatchAddGlossaryTermModalVisible && ( + setIsBatchAddGlossaryTermModalVisible(false)} + fixedEntityTypes={Array.from( + entityRegistry.getTypesWithSupportedCapabilities(EntityCapabilityType.GLOSSARY_TERMS), + )} + /> + )} + {isBatchSetDomainModalVisible && ( + setIsBatchSetDomainModalVisible(false)} + fixedEntityTypes={Array.from( + entityRegistry.getTypesWithSupportedCapabilities(EntityCapabilityType.DOMAINS), + )} + /> + )} + + ); +} + +export default EntityActions; diff --git a/datahub-web-react/src/app/entity/shared/siblingUtils.ts b/datahub-web-react/src/app/entity/shared/siblingUtils.ts new file mode 100644 index 00000000000000..c0076a1829ce06 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/siblingUtils.ts @@ -0,0 +1,208 @@ +import merge from 'deepmerge'; +import { unionBy } from 'lodash'; +import { useLocation } from 'react-router-dom'; +import * as QueryString from 'query-string'; +import { Entity, MatchedField, Maybe, SiblingProperties } from '../../../types.generated'; + +export function stripSiblingsFromEntity(entity: any) { + return { + ...entity, + siblings: null, + siblingPlatforms: null, + }; +} +function cleanHelper(obj, visited) { + if (visited.has(obj)) return obj; + visited.add(obj); + + const object = obj; + Object.entries(object).forEach(([k, v]) => { + if (v && typeof v === 'object') { + cleanHelper(v, visited); + } + if ((v && typeof v === 'object' && !Object.keys(v).length) || v === null || v === undefined || v === '') { + if (Array.isArray(object)) { + object.splice(Number(k), 1); + } else { + delete object[k]; + } + } + }); + return object; +} + +function clean(obj) { + const visited = new Set(); + return cleanHelper(obj, visited); +} + +const combineMerge = (target, source, options) => { + const destination = target.slice(); + + source.forEach((item, index) => { + if (typeof destination[index] === 'undefined') { + destination[index] = options.cloneUnlessOtherwiseSpecified(item, options); + } else if (options.isMergeableObject(item)) { + destination[index] = merge(target[index], item, options); + } else if (target.indexOf(item) === -1) { + destination.push(item); + } + }); + return destination; +}; + +const mergeTags = (destinationArray, sourceArray, _options) => { + return unionBy(destinationArray, sourceArray, 'tag.urn'); +}; + +const mergeTerms = (destinationArray, sourceArray, _options) => { + return unionBy(destinationArray, sourceArray, 'term.urn'); +}; + +const mergeAssertions = (destinationArray, sourceArray, _options) => { + return unionBy(destinationArray, sourceArray, 'urn'); +}; + +const mergeProperties = (destinationArray, sourceArray, _options) => { + return unionBy(destinationArray, sourceArray, 'key'); +}; + +const mergeOwners = (destinationArray, sourceArray, _options) => { + return unionBy(destinationArray, sourceArray, 'owner.urn'); +}; + +function getArrayMergeFunction(key) { + switch (key) { + case 'tags': + return mergeTags; + case 'terms': + return mergeTerms; + case 'assertions': + return mergeAssertions; + case 'customProperties': + return mergeProperties; + case 'owners': + return mergeOwners; + default: + return undefined; + } +} + +const customMerge = (isPrimary, key) => { + if (key === 'upstream' || key === 'downstream') { + return (_secondary, primary) => primary; + } + if (key === 'platform') { + return (secondary, primary) => (isPrimary ? primary : secondary); + } + if (key === 'tags' || key === 'terms' || key === 'assertions' || key === 'customProperties' || key === 'owners') { + return (secondary, primary) => { + return merge(secondary, primary, { + arrayMerge: getArrayMergeFunction(key), + customMerge: customMerge.bind({}, isPrimary), + }); + }; + } + return (secondary, primary) => { + return merge(secondary, primary, { + arrayMerge: combineMerge, + customMerge: customMerge.bind({}, isPrimary), + }); + }; +}; + +export const getEntitySiblingData = (baseEntity: T): Maybe => { + if (!baseEntity) { + return null; + } + const baseEntityKey = Object.keys(baseEntity)[0]; + const extractedBaseEntity = baseEntity[baseEntityKey]; + + // eslint-disable-next-line @typescript-eslint/dot-notation + return extractedBaseEntity?.['siblings']; +}; + +export const combineEntityDataWithSiblings = (baseEntity: T): T => { + if (!baseEntity) { + return baseEntity; + } + const baseEntityKey = Object.keys(baseEntity)[0]; + const extractedBaseEntity = baseEntity[baseEntityKey]; + + // eslint-disable-next-line @typescript-eslint/dot-notation + const siblingAspect = extractedBaseEntity.siblings; + if ((siblingAspect?.siblings || []).length === 0) { + return baseEntity; + } + + // eslint-disable-next-line @typescript-eslint/dot-notation + const siblings: T[] = siblingAspect?.siblings || []; + const isPrimary = !!extractedBaseEntity?.siblings?.isPrimary; + + const combinedBaseEntity: any = siblings.reduce( + (prev, current) => + merge(clean(isPrimary ? current : prev), clean(isPrimary ? prev : current), { + arrayMerge: combineMerge, + customMerge: customMerge.bind({}, isPrimary), + }), + extractedBaseEntity, + ) as T; + + // Force the urn of the combined entity to the current entity urn. + combinedBaseEntity.urn = extractedBaseEntity.urn; + + return { [baseEntityKey]: combinedBaseEntity } as unknown as T; +}; + +export type CombinedSearchResult = { + entity: Entity; + matchedFields: MatchedField[]; + matchedEntities?: Entity[]; +}; + +export function combineSiblingsInSearchResults( + results: + | { + entity: Entity; + matchedFields: MatchedField[]; + }[] + | undefined, +) { + const combinedResults: CombinedSearchResult[] | undefined = []; + const siblingsToPair: Record = {}; + + // set sibling associations + results?.forEach((result) => { + if (result.entity.urn in siblingsToPair) { + // filter from repeating + // const siblingsCombinedResult = siblingsToPair[result.entity.urn]; + // siblingsCombinedResult.matchedEntities?.push(result.entity); + return; + } + + const combinedResult: CombinedSearchResult = result; + const { entity }: { entity: any } = result; + const siblingUrns = entity?.siblings?.siblings?.map((sibling) => sibling.urn) || []; + if (siblingUrns.length > 0) { + combinedResult.matchedEntities = entity.siblings.isPrimary + ? [stripSiblingsFromEntity(entity), ...entity.siblings.siblings] + : [...entity.siblings.siblings, stripSiblingsFromEntity(entity)]; + siblingUrns.forEach((urn) => { + siblingsToPair[urn] = combinedResult; + }); + } + combinedResults.push(combinedResult); + }); + + return combinedResults; +} +// used to determine whether sibling entities should be shown merged or not +export const SEPARATE_SIBLINGS_URL_PARAM = 'separate_siblings'; + +// used to determine whether sibling entities should be shown merged or not +export function useIsSeparateSiblingsMode() { + const location = useLocation(); + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + + return params[SEPARATE_SIBLINGS_URL_PARAM] === 'true'; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx index 7a69002b87b312..fa2edd32f04d45 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx @@ -2,10 +2,7 @@ import { Empty } from 'antd'; import React, { useEffect, useMemo, useState } from 'react'; import styled from 'styled-components'; import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated'; -import { - useGetSchemaBlameQuery, - useGetSchemaBlameVersionsQuery, -} from '../../../../../../graphql/schemaBlame.generated'; +import { useGetSchemaBlameQuery, useGetSchemaVersionListQuery } from '../../../../../../graphql/schemaBlame.generated'; import SchemaEditableContext from '../../../../../shared/SchemaEditableContext'; import SchemaHeader from '../../../../dataset/profile/schema/components/SchemaHeader'; import SchemaRawView from '../../../../dataset/profile/schema/components/SchemaRawView'; @@ -13,9 +10,7 @@ import { KEY_SCHEMA_PREFIX } from '../../../../dataset/profile/schema/utils/cons import { groupByFieldPath } from '../../../../dataset/profile/schema/utils/utils'; import { ANTD_GRAY } from '../../../constants'; import { useBaseEntity, useEntityData } from '../../../EntityContext'; -import { ChangeCategoryType, SchemaFieldBlame, SemanticVersionStruct } from '../../../../../../types.generated'; -import { toLocalDateTimeString } from '../../../../../shared/time/timeUtils'; -import { SchemaViewType } from '../../../../dataset/profile/schema/utils/types'; +import { SchemaFieldBlame, SemanticVersionStruct } from '../../../../../../types.generated'; import SchemaTable from './SchemaTable'; import useGetSemanticVersionFromUrlParams from './utils/useGetSemanticVersionFromUrlParams'; import { useGetVersionedDatasetQuery } from '../../../../../../graphql/versionedDataset.generated'; @@ -55,22 +50,20 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { ); const [showKeySchema, setShowKeySchema] = useState(false); - const [schemaViewMode, setSchemaViewMode] = useState(SchemaViewType.NORMAL); + const [showSchemaAuditView, setShowSchemaAuditView] = useState(false); - const { data: getSchemaBlameVersionsData } = useGetSchemaBlameVersionsQuery({ + const { data: getSchemaVersionListData } = useGetSchemaVersionListQuery({ skip: !datasetUrn, variables: { input: { datasetUrn, - categories: [ChangeCategoryType.TechnicalSchema], }, }, }); - const latestVersion: string = getSchemaBlameVersionsData?.getSchemaBlame?.latestVersion?.semanticVersion || ''; + const latestVersion: string = getSchemaVersionListData?.getSchemaVersionList?.latestVersion?.semanticVersion || ''; - const showSchemaBlame: boolean = schemaViewMode === SchemaViewType.BLAME; const versionList: Array = - getSchemaBlameVersionsData?.getSchemaBlame?.semanticVersionList || []; + getSchemaVersionListData?.getSchemaVersionList?.semanticVersionList || []; const version = useGetSemanticVersionFromUrlParams(); const selectedVersion = version || latestVersion; @@ -78,9 +71,10 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { (semanticVersion) => semanticVersion.semanticVersion === selectedVersion, ); const selectedVersionStamp: string = selectedSemanticVersionStruct?.versionStamp || ''; + const isVersionLatest = selectedVersion === latestVersion; let editMode = true; - if (selectedVersion !== latestVersion) { + if (!isVersionLatest) { editMode = false; } else if (properties && properties.hasOwnProperty('editMode')) { editMode = properties.editMode; @@ -92,7 +86,6 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { input: { datasetUrn, version: selectedVersion, - categories: [ChangeCategoryType.TechnicalSchema], }, }, }); @@ -120,11 +113,8 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { return groupByFieldPath(schemaMetadata?.fields, { showKeySchema }); }, [schemaMetadata, showKeySchema]); - const lastUpdatedTimeString = `Reported at ${ - (getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp && - toLocalDateTimeString(getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp)) || - 'unknown' - }`; + const lastUpdated = getSchemaBlameData?.getSchemaBlame?.version?.semanticVersionTimestamp; + const lastObserved = versionedDatasetData.data?.versionedDataset?.schema?.lastObserved; const schemaFieldBlameList: Array = (getSchemaBlameData?.getSchemaBlame?.schemaFieldBlameList as Array) || []; @@ -139,11 +129,12 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { hasKeySchema={hasKeySchema} showKeySchema={showKeySchema} setShowKeySchema={setShowKeySchema} - lastUpdatedTimeString={lastUpdatedTimeString} + lastObserved={lastObserved} + lastUpdated={lastUpdated} selectedVersion={selectedVersion} versionList={versionList} - schemaView={schemaViewMode} - setSchemaView={setSchemaViewMode} + showSchemaAuditView={showSchemaAuditView} + setShowSchemaAuditView={setShowSchemaAuditView} /> {/* eslint-disable-next-line no-nested-ternary */} {showRaw ? ( @@ -162,7 +153,7 @@ export const SchemaTab = ({ properties }: { properties?: any }) => { editableSchemaMetadata={editableSchemaMetadata} usageStats={usageStats} schemaFieldBlameList={schemaFieldBlameList} - showSchemaBlame={showSchemaBlame} + showSchemaAuditView={showSchemaAuditView} /> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx index 4ddc35bca49db6..504675a64bbf13 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx @@ -44,7 +44,7 @@ export type Props = { editMode?: boolean; usageStats?: UsageQueryResult | null; schemaFieldBlameList?: Array | null; - showSchemaBlame: boolean; + showSchemaAuditView: boolean; }; export default function SchemaTable({ rows, @@ -53,7 +53,7 @@ export default function SchemaTable({ usageStats, editMode = true, schemaFieldBlameList, - showSchemaBlame, + showSchemaAuditView, }: Props): JSX.Element { const hasUsageStats = useMemo(() => (usageStats?.aggregations?.fields?.length || 0) > 0, [usageStats]); @@ -149,7 +149,7 @@ export default function SchemaTable({ allColumns = [...allColumns, usageColumn]; } - if (showSchemaBlame) { + if (showSchemaAuditView) { allColumns = [...allColumns, blameColumn]; } diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx index 737b06030aa695..f6736c4e2035e3 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx @@ -1,12 +1,13 @@ import React from 'react'; +import DOMPurify from 'dompurify'; import { EditableSchemaMetadata, SchemaField, SubResourceType } from '../../../../../../../types.generated'; import DescriptionField from '../../../../../dataset/profile/schema/components/SchemaDescriptionField'; import { pathMatchesNewPath } from '../../../../../dataset/profile/schema/utils/utils'; import { useUpdateDescriptionMutation } from '../../../../../../../graphql/mutations.generated'; -import { useEntityData, useRefetch } from '../../../../EntityContext'; +import { useMutationUrn, useRefetch } from '../../../../EntityContext'; export default function useDescriptionRenderer(editableSchemaMetadata: EditableSchemaMetadata | null | undefined) { - const { urn } = useEntityData(); + const urn = useMutationUrn(); const refetch = useRefetch(); const [updateDescription] = useUpdateDescriptionMutation(); @@ -14,17 +15,20 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find( (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), ); + const displayedDescription = relevantEditableFieldInfo?.description || description; + const sanitizedDescription = DOMPurify.sanitize(displayedDescription); + const original = record.description ? DOMPurify.sanitize(record.description) : undefined; return ( updateDescription({ variables: { input: { - description: updatedDescription, + description: DOMPurify.sanitize(updatedDescription), resourceUrn: urn, subResource: record.fieldPath, subResourceType: SubResourceType.DatasetField, diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx index c2c46be8a6b98b..5f1190ec347c19 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { EditableSchemaMetadata, EntityType, GlobalTags, SchemaField } from '../../../../../../../types.generated'; import TagTermGroup from '../../../../../../shared/tags/TagTermGroup'; import { pathMatchesNewPath } from '../../../../../dataset/profile/schema/utils/utils'; -import { useEntityData, useRefetch } from '../../../../EntityContext'; +import { useMutationUrn, useRefetch } from '../../../../EntityContext'; export default function useTagsAndTermsRenderer( editableSchemaMetadata: EditableSchemaMetadata | null | undefined, @@ -10,7 +10,7 @@ export default function useTagsAndTermsRenderer( setTagHoveredIndex: (index: string | undefined) => void, options: { showTags: boolean; showTerms: boolean }, ) { - const { urn } = useEntityData(); + const urn = useMutationUrn(); const refetch = useRefetch(); const tagAndTermRender = (tags: GlobalTags, record: SchemaField, rowIndex: number | undefined) => { diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/ColumnStats.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/ColumnStats.tsx index f6637a507c8a16..f438ce104714ab 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/ColumnStats.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/ColumnStats.tsx @@ -1,10 +1,11 @@ -import { Tag, Typography } from 'antd'; +import { Typography } from 'antd'; import { ColumnsType, ColumnType } from 'antd/lib/table'; import React, { useMemo } from 'react'; import styled from 'styled-components'; import { DatasetFieldProfile } from '../../../../../../../types.generated'; import { StyledTable } from '../../../../components/styled/StyledTable'; import { ANTD_GRAY } from '../../../../constants'; +import SampleValueTag from './SampleValueTag'; type Props = { columnStats: Array; @@ -128,7 +129,7 @@ export default function ColumnStats({ columnStats }: Props) { (sampleValues && sampleValues .slice(0, sampleValues.length < 3 ? sampleValues?.length : 3) - .map((value) => {value})) || + .map((value) => )) || unknownValue() ); }, diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/SampleValueTag.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/SampleValueTag.tsx new file mode 100644 index 00000000000000..3f2df14eaa900b --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/SampleValueTag.tsx @@ -0,0 +1,35 @@ +import React, { useState } from 'react'; +import { Tag, Tooltip } from 'antd'; +import styled from 'styled-components'; + +const StyledTag = styled(Tag)` + cursor: pointer; + max-width: 250px; + overflow: hidden; + text-overflow: ellipsis; + + &&:hover { + color: ${(props) => props.theme.styles['primary-color']}; + border-color: ${(props) => props.theme.styles['primary-color']}; + } +`; + +type Props = { + value: string; +}; + +export default function SampleValueTag({ value }: Props) { + const [copied, setCopied] = useState(false); + + const onClick = () => { + setCopied(true); + navigator.clipboard.writeText(value); + setTimeout(() => setCopied(false), 2000); + }; + + return ( + + {value} + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx index 31ee3ded969d6e..ff821698efdfc9 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Stats/snapshot/TableStats.tsx @@ -1,11 +1,11 @@ import { Tooltip, Typography } from 'antd'; import React from 'react'; import styled from 'styled-components'; -import { Maybe, UserUsageCounts } from '../../../../../../../types.generated'; -import UsageFacepile from '../../../../../dataset/profile/UsageFacepile'; +import { CorpUser, Maybe, UserUsageCounts } from '../../../../../../../types.generated'; import { InfoItem } from '../../../../components/styled/InfoItem'; import { ANTD_GRAY } from '../../../../constants'; import { countFormatter, countSeparator } from '../../../../../../../utils/formatter/index'; +import { ExpandedActorGroup } from '../../../../components/styled/ExpandedActorGroup'; type Props = { rowCount?: number; @@ -47,7 +47,7 @@ export default function TableStats({ {rowCount && ( - + {countFormatter(rowCount)} @@ -67,10 +67,20 @@ export default function TableStats({ )} - {users && ( + {users && users.length > 0 && (
- + user && user?.user !== undefined && user?.user !== null) + .map((user) => user?.user as CorpUser) || [] + } + max={4} + />
)} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDescription.tsx index c69fa8b6edc7ca..c9c81c0bcbbd58 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDescription.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDescription.tsx @@ -1,5 +1,5 @@ -import { Popover, Typography } from 'antd'; -import React from 'react'; +import { Popover, Typography, Button } from 'antd'; +import React, { useState } from 'react'; import styled from 'styled-components'; import { AssertionStdAggregation, @@ -10,10 +10,11 @@ import { SchemaFieldRef, } from '../../../../../../types.generated'; import { getFormattedParameterValue } from './assertionUtils'; +import { DatasetAssertionLogicModal } from './DatasetAssertionLogicModal'; -const SqlText = styled.pre` - margin: 0px; +const ViewLogicButton = styled(Button)` padding: 0px; + margin: 0px; `; type Props = { @@ -44,7 +45,8 @@ const getSchemaAggregationText = ( ); } default: - throw new Error(`Unsupported schema aggregation assertion ${aggregation} provided.`); + console.error(`Unsupported schema aggregation assertion ${aggregation} provided.`); + return Dataset columns are; } }; @@ -61,7 +63,8 @@ const getRowsAggregationText = (aggregation: AssertionStdAggregation | undefined case AssertionStdAggregation.Native: return Dataset rows are; default: - throw new Error(`Unsupported Dataset Rows Aggregation ${aggregation} provided`); + console.error(`Unsupported Dataset Rows Aggregation ${aggregation} provided`); + return Dataset rows are; } }; @@ -73,36 +76,38 @@ const getColumnAggregationText = ( aggregation: AssertionStdAggregation | undefined | null, field: SchemaFieldRef | undefined, ) => { + let columnText = field?.path; if (field === undefined) { - throw new Error(`Invalid field provided for Dataset Assertion with scope Column ${JSON.stringify(field)}`); + columnText = 'undefined'; + console.error(`Invalid field provided for Dataset Assertion with scope Column ${JSON.stringify(field)}`); } switch (aggregation) { // Hybrid Aggregations case AssertionStdAggregation.UniqueCount: { return ( - Unique value count for column {field.path} is + Unique value count for column {columnText} is ); } case AssertionStdAggregation.UniquePropotion: { return ( - Unique value proportion for column {field.path} is + Unique value proportion for column {columnText} is ); } case AssertionStdAggregation.NullCount: { return ( - Null count for column {field.path} is + Null count for column {columnText} is ); } case AssertionStdAggregation.NullProportion: { return ( - Null proportion for column {field.path} is + Null proportion for column {columnText} is ); } @@ -110,35 +115,35 @@ const getColumnAggregationText = ( case AssertionStdAggregation.Min: { return ( - Minimum value for column {field.path} is + Minimum value for column {columnText} is ); } case AssertionStdAggregation.Max: { return ( - Maximum value for column {field.path} is + Maximum value for column {columnText} is ); } case AssertionStdAggregation.Mean: { return ( - Mean value for column {field.path} is + Mean value for column {columnText} is ); } case AssertionStdAggregation.Median: { return ( - Median value for column {field.path} is + Median value for column {columnText} is ); } case AssertionStdAggregation.Stddev: { return ( - Standard deviation for column {field.path} is + Standard deviation for column {columnText} is ); } @@ -146,7 +151,7 @@ const getColumnAggregationText = ( case AssertionStdAggregation.Native: { return ( - Column {field.path} values are + Column {columnText} values are ); } @@ -154,7 +159,7 @@ const getColumnAggregationText = ( // No aggregation on the column at hand. Treat the column as a set of values. return ( - Column {field.path} values are + Column {columnText} values are ); } @@ -176,7 +181,8 @@ const getAggregationText = ( case DatasetAssertionScope.DatasetColumn: return getColumnAggregationText(aggregation, fields?.length === 1 ? fields[0] : undefined); default: - throw new Error(`Unsupported Dataset Assertion scope ${scope} provided`); + console.error(`Unsupported Dataset Assertion scope ${scope} provided`); + return 'Dataset is'; } }; @@ -314,7 +320,7 @@ const TOOLTIP_MAX_WIDTH = 440; */ export const DatasetAssertionDescription = ({ assertionInfo }: Props) => { const { scope, aggregation, fields, operator, parameters, nativeType, nativeParameters, logic } = assertionInfo; - + const [isLogicVisible, setIsLogicVisible] = useState(false); /** * Build a description component from a) input (aggregation, inputs) b) the operator text */ @@ -342,7 +348,19 @@ export const DatasetAssertionDescription = ({ assertionInfo }: Props) => { } > -
{(logic && {logic}) || description}
+
{description}
+ {logic && ( +
+ setIsLogicVisible(true)} type="link"> + View Logic + +
+ )} + setIsLogicVisible(false)} + /> ); }; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionLogicModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionLogicModal.tsx new file mode 100644 index 00000000000000..549a10ecd24cde --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionLogicModal.tsx @@ -0,0 +1,24 @@ +import { Modal, Button } from 'antd'; +import React from 'react'; +import Query from '../Queries/Query'; + +export type AssertionsSummary = { + totalAssertions: number; + totalRuns: number; + failedRuns: number; + succeededRuns: number; +}; + +type Props = { + logic: string; + visible: boolean; + onClose: () => void; +}; + +export const DatasetAssertionLogicModal = ({ logic, visible, onClose }: Props) => { + return ( + Close}> + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx index f43fbf07aacdb2..13dcfd34b8219e 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx @@ -77,9 +77,9 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { type: assertion.info?.type, platform: assertion.platform, datasetAssertionInfo: assertion.info?.datasetAssertion, - lastExecTime: assertion.runEvents?.runEvents.length && assertion.runEvents.runEvents[0].timestampMillis, + lastExecTime: assertion.runEvents?.runEvents?.length && assertion.runEvents.runEvents[0].timestampMillis, lastExecResult: - assertion.runEvents?.runEvents.length && + assertion.runEvents?.runEvents?.length && assertion.runEvents.runEvents[0].status === AssertionRunStatus.Complete && assertion.runEvents.runEvents[0].result?.type, })); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsSummary.tsx index 6a890ff9d7ec98..e61358f3f1c02c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsSummary.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsSummary.tsx @@ -6,9 +6,9 @@ import { ANTD_GRAY } from '../../../constants'; const SummaryHeader = styled.div` width: 100%; - height: 80px; padding-left: 40px; - padding-top: 0px; + padding-top: 20px; + padding-bottom: 20px; display: flex; align-items: center; border-bottom: 1px solid ${ANTD_GRAY[4.5]}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx index 46ab1e9d14bdb3..ccfb6a3fdfc772 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx @@ -9,6 +9,7 @@ import { DatasetAssertionsList } from './DatasetAssertionsList'; import { DatasetAssertionsSummary } from './DatasetAssertionsSummary'; import { sortAssertions } from './assertionUtils'; import { TestResults } from './TestResults'; +import { combineEntityDataWithSiblings, useIsSeparateSiblingsMode } from '../../../siblingUtils'; /** * Returns a status summary for the assertions associated with a Dataset. @@ -21,9 +22,9 @@ const getAssertionsStatusSummary = (assertions: Array) => { totalAssertions: assertions.length, }; assertions.forEach((assertion) => { - if (assertion.runEvents?.runEvents.length) { - const mostRecentRun = assertion.runEvents?.runEvents[0]; - const resultType = mostRecentRun.result?.type; + if ((assertion.runEvents?.runEvents?.length || 0) > 0) { + const mostRecentRun = assertion.runEvents?.runEvents?.[0]; + const resultType = mostRecentRun?.result?.type; if (AssertionResultType.Success === resultType) { summary.succeededRuns++; } @@ -47,23 +48,32 @@ enum ViewType { export const ValidationsTab = () => { const { urn, entityData } = useEntityData(); const { data, refetch } = useGetDatasetAssertionsQuery({ variables: { urn } }); + const isHideSiblingMode = useIsSeparateSiblingsMode(); + + const combinedData = isHideSiblingMode ? data : combineEntityDataWithSiblings(data); + const [removedUrns, setRemovedUrns] = useState([]); /** * Determines which view should be visible: assertions or tests. */ const [view, setView] = useState(ViewType.ASSERTIONS); - const assertions = (data && data.dataset?.assertions?.assertions?.map((assertion) => assertion as Assertion)) || []; - const totalAssertions = data?.dataset?.assertions?.total || 0; + const assertions = + (combinedData && combinedData.dataset?.assertions?.assertions?.map((assertion) => assertion as Assertion)) || + []; + const filteredAssertions = assertions.filter((assertion) => !removedUrns.includes(assertion.urn)); + const numAssertions = filteredAssertions.length; const passingTests = (entityData as any)?.testResults?.passing || []; const maybeFailingTests = (entityData as any)?.testResults?.failing || []; const totalTests = maybeFailingTests.length + passingTests.length; useEffect(() => { - if (totalAssertions === 0) { + if (totalTests > 0 && numAssertions === 0) { setView(ViewType.TESTS); + } else { + setView(ViewType.ASSERTIONS); } - }, [totalAssertions]); + }, [totalTests, numAssertions]); // Pre-sort the list of assertions based on which has been most recently executed. assertions.sort(sortAssertions); @@ -72,9 +82,9 @@ export const ValidationsTab = () => { <>
- handleEditorChange(v || '')} preview="live" visiableDragbar={false} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx index 997dccec6c8c16..aa1128646801ac 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { Link } from 'react-router-dom'; -import styled from 'styled-components'; +import styled from 'styled-components/macro'; import { message, Button, List, Typography } from 'antd'; import { LinkOutlined, DeleteOutlined } from '@ant-design/icons'; import { EntityType } from '../../../../../../types.generated'; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx new file mode 100644 index 00000000000000..62d612b0df4df2 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/DashboardDatasetsTab.tsx @@ -0,0 +1,19 @@ +import React from 'react'; +import { useBaseEntity } from '../../EntityContext'; +import { EntityType } from '../../../../../types.generated'; +import { EntityList } from './components/EntityList'; +import { useEntityRegistry } from '../../../../useEntityRegistry'; + +export const DashboardDatasetsTab = () => { + const entity = useBaseEntity() as any; + const dashboard = entity && entity.dashboard; + const datasets = dashboard?.datasets?.relationships.map((relationship) => relationship.entity); + const entityRegistry = useEntityRegistry(); + const totalDatasets = dashboard?.datasets?.total || 0; + const title = `Consumes ${totalDatasets} ${ + totalDatasets === 1 + ? entityRegistry.getEntityName(EntityType.Dataset) + : entityRegistry.getCollectionName(EntityType.Dataset) + }`; + return ; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DataJobFlowTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DataJobFlowTab.tsx index b5eb3896aaecc8..f842f1adb9c904 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Entity/DataJobFlowTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/DataJobFlowTab.tsx @@ -7,8 +7,8 @@ import { useEntityRegistry } from '../../../../useEntityRegistry'; export const DataJobFlowTab = () => { const entity = useBaseEntity() as any; const dataJob = entity && entity.dataJob; - const dataFlows = dataJob?.parentFlow?.relationships.map((relationship) => relationship.entity); + const dataFlow = dataJob?.dataFlow; const entityRegistry = useEntityRegistry(); const title = `Part of ${entityRegistry.getEntityName(EntityType.DataFlow)}`; - return ; + return ; }; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx index e15995b0ae60fa..5993df25bbef67 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Lineage/ImpactAnalysis.tsx @@ -9,8 +9,8 @@ import { ENTITY_FILTER_NAME } from '../../../../search/utils/constants'; import useFilters from '../../../../search/utils/useFilters'; import { SearchCfg } from '../../../../../conf'; import analytics, { EventType } from '../../../../analytics'; -import { EmbeddedListSearch } from '../../components/styled/search/EmbeddedListSearch'; import generateUseSearchResultsViaRelationshipHook from './generateUseSearchResultsViaRelationshipHook'; +import { EmbeddedListSearchSection } from '../../components/styled/search/EmbeddedListSearchSection'; const ImpactAnalysisWrapper = styled.div` flex: 1; @@ -18,9 +18,10 @@ const ImpactAnalysisWrapper = styled.div` type Props = { urn: string; + direction: LineageDirection; }; -export const ImpactAnalysis = ({ urn }: Props) => { +export const ImpactAnalysis = ({ urn, direction }: Props) => { const location = useLocation(); const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); @@ -38,7 +39,7 @@ export const ImpactAnalysis = ({ urn }: Props) => { variables: { input: { urn, - direction: LineageDirection.Downstream, + direction, types: entityFilters, query, start: (page - 1) * SearchCfg.RESULTS_PER_PAGE, @@ -60,11 +61,13 @@ export const ImpactAnalysis = ({ urn }: Props) => { return ( - ); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx index 70f13e7c21b098..9bf15153e57254 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Lineage/LineageTab.tsx @@ -1,71 +1,66 @@ import React, { useCallback, useState } from 'react'; import { Button } from 'antd'; import { useHistory } from 'react-router'; -import { BarsOutlined, PartitionOutlined } from '@ant-design/icons'; -import { VscGraphLeft } from 'react-icons/vsc'; -import styled from 'styled-components'; +import { ArrowDownOutlined, ArrowUpOutlined, PartitionOutlined } from '@ant-design/icons'; +import styled from 'styled-components/macro'; -import { useEntityData, useLineageData } from '../../EntityContext'; +import { useEntityData } from '../../EntityContext'; import TabToolbar from '../../components/styled/TabToolbar'; import { getEntityPath } from '../../containers/profile/utils'; import { useEntityRegistry } from '../../../../useEntityRegistry'; -import { LineageTable } from './LineageTable'; import { ImpactAnalysis } from './ImpactAnalysis'; -import { useAppConfig } from '../../../../useAppConfig'; +import { LineageDirection } from '../../../../../types.generated'; -const ImpactAnalysisIcon = styled(VscGraphLeft)` - transform: scaleX(-1); - font-size: 18px; +const StyledTabToolbar = styled(TabToolbar)` + justify-content: space-between; +`; + +const StyledButton = styled(Button)<{ isSelected: boolean }>` + ${(props) => + props.isSelected && + ` + color: #1890ff; + &:focus { + color: #1890ff; + } + `} `; export const LineageTab = () => { const { urn, entityType } = useEntityData(); const history = useHistory(); const entityRegistry = useEntityRegistry(); - const lineage = useLineageData(); - const [showImpactAnalysis, setShowImpactAnalysis] = useState(false); - const appConfig = useAppConfig(); + const [lineageDirection, setLineageDirection] = useState(LineageDirection.Downstream); const routeToLineage = useCallback(() => { - history.push(getEntityPath(entityType, urn, entityRegistry, true)); + history.push(getEntityPath(entityType, urn, entityRegistry, true, false)); }, [history, entityType, urn, entityRegistry]); - const upstreamEntities = lineage?.upstreamChildren?.map((result) => result.entity); - const downstreamEntities = lineage?.downstreamChildren?.map((result) => result.entity); return ( <> - +
- - {appConfig.config.lineageConfig.supportsImpactAnalysis && - (showImpactAnalysis ? ( - - ) : ( - - ))} + setLineageDirection(LineageDirection.Downstream)} + > + Downstream + + setLineageDirection(LineageDirection.Upstream)} + > + Upstream +
- - {showImpactAnalysis ? ( - - ) : ( - <> - - - - )} + + + ); }; diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index 92354aa9eb1f7f..76f366e3f19341 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -17,9 +17,7 @@ import { Ownership, OwnershipUpdate, SchemaMetadata, - StringMapEntry, EntityLineageResult, - Domain, SubTypes, Container, Health, @@ -29,6 +27,9 @@ import { ParentContainersResult, EntityRelationshipsResult, ParentNodesResult, + SiblingProperties, + CustomPropertiesEntry, + DomainAssociation, } from '../../../types.generated'; import { FetchedEntity } from '../../lineage/types'; @@ -50,20 +51,26 @@ export type EntitySidebarSection = { properties?: any; }; +export type EntitySubHeaderSection = { + component: React.FunctionComponent<{ properties?: any }>; +}; + export type GenericEntityProperties = { urn?: string; name?: Maybe; properties?: Maybe<{ description?: Maybe; qualifiedName?: Maybe; + sourceUrl?: Maybe; + sourceRef?: Maybe; }>; globalTags?: Maybe; glossaryTerms?: Maybe; ownership?: Maybe; - domain?: Maybe; + domain?: Maybe; platform?: Maybe; dataPlatformInstance?: Maybe; - customProperties?: Maybe; + customProperties?: Maybe; institutionalMemory?: Maybe; schemaMetadata?: Maybe; externalUrl?: Maybe; @@ -78,13 +85,15 @@ export type GenericEntityProperties = { subTypes?: Maybe; entityCount?: number; container?: Maybe; - health?: Maybe; + health?: Maybe>; status?: Maybe; deprecation?: Maybe; parentContainers?: Maybe; children?: Maybe; parentNodes?: Maybe; isAChildren?: Maybe; + siblings?: Maybe; + siblingPlatforms?: Maybe; }; export type GenericEntityUpdate = { @@ -111,6 +120,7 @@ export type UpdateEntityType = ( export type EntityContextType = { urn: string; entityType: EntityType; + dataNotCombinedWithSiblings: any; entityData: GenericEntityProperties | null; baseEntity: any; updateEntity?: UpdateEntityType | null; @@ -122,3 +132,8 @@ export type EntityContextType = { export type RequiredAndNotNull = { [P in keyof T]-?: Exclude; }; + +export type EntityAndType = { + urn: string; + type: EntityType; +}; diff --git a/datahub-web-react/src/app/entity/shared/utils.ts b/datahub-web-react/src/app/entity/shared/utils.ts index 0b50b6c8cc9f48..065115b56ce2bc 100644 --- a/datahub-web-react/src/app/entity/shared/utils.ts +++ b/datahub-web-react/src/app/entity/shared/utils.ts @@ -1,5 +1,11 @@ import { GenericEntityProperties } from './types'; +export function dictToQueryStringParams(params: Record) { + return Object.keys(params) + .map((key) => `${key}=${params[key]}`) + .join('&'); +} + export function urlEncodeUrn(urn: string) { return ( urn && @@ -65,3 +71,12 @@ export function getPlatformName(entityData: GenericEntityProperties | null) { } export const EDITED_DESCRIPTIONS_CACHE_NAME = 'editedDescriptions'; + +export const FORBIDDEN_URN_CHARS_REGEX = /.*[(),\\].*/; + +/** + * Utility function for checking whether a list is a subset of another. + */ +export const isListSubset = (l1, l2): boolean => { + return l1.every((result) => l2.indexOf(result) >= 0); +}; diff --git a/datahub-web-react/src/app/entity/tag/Tag.tsx b/datahub-web-react/src/app/entity/tag/Tag.tsx index 57082d324e743d..57a9043953235b 100644 --- a/datahub-web-react/src/app/entity/tag/Tag.tsx +++ b/datahub-web-react/src/app/entity/tag/Tag.tsx @@ -3,7 +3,7 @@ import * as React from 'react'; import styled from 'styled-components'; import { Tag, EntityType, SearchResult } from '../../../types.generated'; import DefaultPreviewCard from '../../preview/DefaultPreviewCard'; -import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { urlEncodeUrn } from '../shared/utils'; import TagProfile from './TagProfile'; @@ -60,6 +60,7 @@ export class TagEntity implements Entity { url={`/${this.getPathName()}/${urlEncodeUrn(data.urn)}`} logoComponent={} type="Tag" + typeIcon={this.icon(14, IconStyleType.ACCENT)} /> ); @@ -74,4 +75,8 @@ export class TagEntity implements Entity { getGenericEntityProperties = (tag: Tag) => { return getDataForEntityType({ data: tag, entityType: this.type, getOverrideProperties: (data) => data }); }; + + supportedCapabilities = () => { + return new Set([EntityCapabilityType.OWNERS]); + }; } diff --git a/datahub-web-react/src/app/entity/user/User.tsx b/datahub-web-react/src/app/entity/user/User.tsx index 931e47c3c5ca28..d5a8ac4b948b4a 100644 --- a/datahub-web-react/src/app/entity/user/User.tsx +++ b/datahub-web-react/src/app/entity/user/User.tsx @@ -67,11 +67,16 @@ export class UserEntity implements Entity { data.properties?.fullName || data.info?.displayName || // Deprecated info field data.info?.fullName || // Deprecated info field - data.username + data.username || + data.urn ); }; getGenericEntityProperties = (user: CorpUser) => { return getDataForEntityType({ data: user, entityType: this.type, getOverrideProperties: (data) => data }); }; + + supportedCapabilities = () => { + return new Set([]); + }; } diff --git a/datahub-web-react/src/app/entity/user/UserAssets.tsx b/datahub-web-react/src/app/entity/user/UserAssets.tsx index c28d4fcedc660f..8a8d4b569871e5 100644 --- a/datahub-web-react/src/app/entity/user/UserAssets.tsx +++ b/datahub-web-react/src/app/entity/user/UserAssets.tsx @@ -1,6 +1,6 @@ import React from 'react'; import styled from 'styled-components'; -import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch'; +import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection'; const UserAssetsWrapper = styled.div` height: calc(100vh - 114px); @@ -14,7 +14,7 @@ type Props = { export const UserAssets = ({ urn }: Props) => { return ( - + sortGlossaryTerms(entityRegistry, termA, termB), + ); + const nodes = nodesData?.getRootGlossaryNodes?.nodes.sort((nodeA, nodeB) => + sortGlossaryNodes(entityRegistry, nodeA, nodeB), + ); - const terms = termsData?.getRootGlossaryTerms?.terms; - const nodes = nodesData?.getRootGlossaryNodes?.nodes; + const hasTermsOrNodes = !!nodes?.length || !!terms?.length; + + const [isCreateTermModalVisible, setIsCreateTermModalVisible] = useState(false); + const [isCreateNodeModalVisible, setIsCreateNodeModalVisible] = useState(false); return ( - + <> + {(termsLoading || nodesLoading) && ( + + )} @@ -62,16 +82,36 @@ function BusinessGlossaryPage() { Glossary - +
+ + +
- + {hasTermsOrNodes && } + {!(termsLoading || nodesLoading) && !hasTermsOrNodes && ( + + )} - + {isCreateTermModalVisible && ( + setIsCreateTermModalVisible(false)} + refetchData={refetchForTerms} + /> + )} + {isCreateNodeModalVisible && ( + setIsCreateNodeModalVisible(false)} + refetchData={refetchForNodes} + /> + )} + ); } diff --git a/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx b/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx new file mode 100644 index 00000000000000..0571e808a5d780 --- /dev/null +++ b/datahub-web-react/src/app/glossary/EmptyGlossarySection.tsx @@ -0,0 +1,69 @@ +import { PlusOutlined } from '@ant-design/icons'; +import { Button, Empty, Typography } from 'antd'; +import React, { useState } from 'react'; +import styled from 'styled-components/macro'; +import { EntityType } from '../../types.generated'; +import CreateGlossaryEntityModal from '../entity/shared/EntityDropdown/CreateGlossaryEntityModal'; + +const StyledEmpty = styled(Empty)` + padding: 80px 40px; + .ant-empty-footer { + .ant-btn:not(:last-child) { + margin-right: 8px; + } + } +`; + +const StyledButton = styled(Button)` + margin-right: 8px; +`; + +interface Props { + refetchForTerms?: () => void; + refetchForNodes?: () => void; +} + +function EmptyGlossarySection(props: Props) { + const { refetchForTerms, refetchForNodes } = props; + + const [isCreateTermModalVisible, setIsCreateTermModalVisible] = useState(false); + const [isCreateNodeModalVisible, setIsCreateNodeModalVisible] = useState(false); + + return ( + <> + + Empty Glossary + + Create Terms and Term Groups to organize data assets using a shared vocabulary. + + + } + > + setIsCreateTermModalVisible(true)}> + Add Term + + setIsCreateNodeModalVisible(true)}> + Add Term Group + + + {isCreateTermModalVisible && ( + setIsCreateTermModalVisible(false)} + refetchData={refetchForTerms} + /> + )} + {isCreateNodeModalVisible && ( + setIsCreateNodeModalVisible(false)} + refetchData={refetchForNodes} + /> + )} + + ); +} + +export default EmptyGlossarySection; diff --git a/datahub-web-react/src/app/glossary/GlossaryBrowser/GlossaryBrowser.tsx b/datahub-web-react/src/app/glossary/GlossaryBrowser/GlossaryBrowser.tsx index 425026e6172cb4..d5303061e5f8db 100644 --- a/datahub-web-react/src/app/glossary/GlossaryBrowser/GlossaryBrowser.tsx +++ b/datahub-web-react/src/app/glossary/GlossaryBrowser/GlossaryBrowser.tsx @@ -2,6 +2,9 @@ import React, { useEffect } from 'react'; import styled from 'styled-components/macro'; import { useGetRootGlossaryNodesQuery, useGetRootGlossaryTermsQuery } from '../../../graphql/glossary.generated'; import { GlossaryNode, GlossaryTerm } from '../../../types.generated'; +import { sortGlossaryNodes } from '../../entity/glossaryNode/utils'; +import { sortGlossaryTerms } from '../../entity/glossaryTerm/utils'; +import { useEntityRegistry } from '../../useEntityRegistry'; import NodeItem from './NodeItem'; import TermItem from './TermItem'; @@ -20,13 +23,23 @@ interface Props { hideTerms?: boolean; openToEntity?: boolean; refreshBrowser?: boolean; + nodeUrnToHide?: string; selectTerm?: (urn: string, displayName: string) => void; selectNode?: (urn: string, displayName: string) => void; } function GlossaryBrowser(props: Props) { - const { rootNodes, rootTerms, isSelecting, hideTerms, refreshBrowser, openToEntity, selectTerm, selectNode } = - props; + const { + rootNodes, + rootTerms, + isSelecting, + hideTerms, + refreshBrowser, + openToEntity, + nodeUrnToHide, + selectTerm, + selectNode, + } = props; const { data: nodesData, refetch: refetchNodes } = useGetRootGlossaryNodesQuery({ skip: !!rootNodes }); const { data: termsData, refetch: refetchTerms } = useGetRootGlossaryTermsQuery({ skip: !!rootTerms }); @@ -34,6 +47,10 @@ function GlossaryBrowser(props: Props) { const displayedNodes = rootNodes || nodesData?.getRootGlossaryNodes?.nodes || []; const displayedTerms = rootTerms || termsData?.getRootGlossaryTerms?.terms || []; + const entityRegistry = useEntityRegistry(); + const sortedNodes = displayedNodes.sort((nodeA, nodeB) => sortGlossaryNodes(entityRegistry, nodeA, nodeB)); + const sortedTerms = displayedTerms.sort((termA, termB) => sortGlossaryTerms(entityRegistry, termA, termB)); + useEffect(() => { if (refreshBrowser) { refetchNodes(); @@ -43,7 +60,7 @@ function GlossaryBrowser(props: Props) { return ( - {displayedNodes.map((node) => ( + {sortedNodes.map((node) => ( ))} {!hideTerms && - displayedTerms.map((term) => ( + sortedTerms.map((term) => ( ))} diff --git a/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx b/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx index 61136928ae53fc..b51093cf677384 100644 --- a/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx +++ b/datahub-web-react/src/app/glossary/GlossaryBrowser/NodeItem.tsx @@ -7,6 +7,8 @@ import { useEntityRegistry } from '../../useEntityRegistry'; import { useGetGlossaryNodeQuery } from '../../../graphql/glossaryNode.generated'; import TermItem, { TermLink as NodeLink, NameWrapper } from './TermItem'; import { useEntityData } from '../../entity/shared/EntityContext'; +import { sortGlossaryNodes } from '../../entity/glossaryNode/utils'; +import { sortGlossaryTerms } from '../../entity/glossaryTerm/utils'; const ItemWrapper = styled.div` display: flex; @@ -49,17 +51,22 @@ interface Props { hideTerms?: boolean; openToEntity?: boolean; refreshBrowser?: boolean; + nodeUrnToHide?: string; selectTerm?: (urn: string, displayName: string) => void; selectNode?: (urn: string, displayName: string) => void; } function NodeItem(props: Props) { - const { node, isSelecting, hideTerms, openToEntity, refreshBrowser, selectTerm, selectNode } = props; + const { node, isSelecting, hideTerms, openToEntity, refreshBrowser, nodeUrnToHide, selectTerm, selectNode } = props; + const shouldHideNode = nodeUrnToHide === node.urn; const [areChildrenVisible, setAreChildrenVisible] = useState(false); const entityRegistry = useEntityRegistry(); const { entityData } = useEntityData(); - const { data } = useGetGlossaryNodeQuery({ variables: { urn: node.urn }, skip: !areChildrenVisible }); + const { data } = useGetGlossaryNodeQuery({ + variables: { urn: node.urn }, + skip: !areChildrenVisible || shouldHideNode, + }); useEffect(() => { if (openToEntity && entityData && entityData.parentNodes?.nodes.some((parent) => parent.urn === node.urn)) { @@ -88,12 +95,16 @@ function NodeItem(props: Props) { const childNodes = (children as any) ?.filter((child) => child.entity?.type === EntityType.GlossaryNode) + .sort((nodeA, nodeB) => sortGlossaryNodes(entityRegistry, nodeA.entity, nodeB.entity)) .map((child) => child.entity) || []; const childTerms = (children as any) ?.filter((child) => child.entity?.type === EntityType.GlossaryTerm) + .sort((termA, termB) => sortGlossaryTerms(entityRegistry, termA.entity, termB.entity)) .map((child) => child.entity) || []; + if (shouldHideNode) return null; + return ( @@ -123,6 +134,7 @@ function NodeItem(props: Props) { isSelecting={isSelecting} hideTerms={hideTerms} openToEntity={openToEntity} + nodeUrnToHide={nodeUrnToHide} selectTerm={selectTerm} selectNode={selectNode} /> diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx index 5a718cc7951220..141e38a7c02fd4 100644 --- a/datahub-web-react/src/app/home/HomePageHeader.tsx +++ b/datahub-web-react/src/app/home/HomePageHeader.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { useHistory } from 'react-router'; import { Typography, Image, Row, Button, Tag } from 'antd'; import styled, { useTheme } from 'styled-components'; @@ -9,14 +9,15 @@ import { useEntityRegistry } from '../useEntityRegistry'; import { navigateToSearchUrl } from '../search/utils/navigateToSearchUrl'; import { SearchBar } from '../search/SearchBar'; import { + GetAutoCompleteMultipleResultsQuery, useGetAutoCompleteMultipleResultsLazyQuery, useGetSearchResultsForMultipleQuery, } from '../../graphql/search.generated'; import { EntityType } from '../../types.generated'; import analytics, { EventType } from '../analytics'; +import { HeaderLinks } from '../shared/admin/HeaderLinks'; import AdhocLink from '../create/AdhocLink'; import HelpLink from '../shared/admin/HelpLink'; -import { AdminHeaderLinks } from '../shared/admin/AdminHeaderLinks'; import { ANTD_GRAY } from '../entity/shared/constants'; import ContactLink from '../shared/admin/ContactLink'; import { useAppConfig } from '../useAppConfig'; @@ -122,6 +123,13 @@ export const HomePageHeader = () => { const user = useGetAuthenticatedUser()?.corpUser; const themeConfig = useTheme(); const appConfig = useAppConfig(); + const [newSuggestionData, setNewSuggestionData] = useState(); + + useEffect(() => { + if (suggestionsData !== undefined) { + setNewSuggestionData(suggestionsData); + } + }, [suggestionsData]); const onSearch = (query: string, type?: EntityType) => { if (!query || query.trim().length === 0) { @@ -146,7 +154,7 @@ export const HomePageHeader = () => { variables: { input: { query, - limit: 30, + limit: 10, }, }, }); @@ -160,7 +168,7 @@ export const HomePageHeader = () => { types: [], query: '*', start: 0, - count: 20, + count: 6, filters: [], }, }, @@ -191,10 +199,10 @@ export const HomePageHeader = () => { )} + - { void; onCreate: (name: string, description: string) => void; }; -export default function CreateGroupModal({ visible, onClose, onCreate }: Props) { +export default function CreateGroupModal({ onClose, onCreate }: Props) { const [stagedName, setStagedName] = useState(''); const [stagedDescription, setStagedDescription] = useState(''); const [stagedId, setStagedId] = useState(undefined); @@ -52,7 +51,7 @@ export default function CreateGroupModal({ visible, onClose, onCreate }: Props) return ( diff --git a/datahub-web-react/src/app/identity/group/GroupList.tsx b/datahub-web-react/src/app/identity/group/GroupList.tsx index aa7eef5c1355b1..1c4a7ffb904b44 100644 --- a/datahub-web-react/src/app/identity/group/GroupList.tsx +++ b/datahub-web-react/src/app/identity/group/GroupList.tsx @@ -122,16 +122,17 @@ export const GroupList = () => { showSizeChanger={false} /> - setIsCreatingGroup(false)} - onCreate={() => { - // Hack to deal with eventual consistency. - setTimeout(function () { - refetch?.(); - }, 2000); - }} - /> + {isCreatingGroup && ( + setIsCreatingGroup(false)} + onCreate={() => { + // Hack to deal with eventual consistency. + setTimeout(function () { + refetch?.(); + }, 2000); + }} + /> + )} ); diff --git a/datahub-web-react/src/app/identity/group/GroupListItem.tsx b/datahub-web-react/src/app/identity/group/GroupListItem.tsx index 30a1f80d962b0f..2679e365f753b3 100644 --- a/datahub-web-react/src/app/identity/group/GroupListItem.tsx +++ b/datahub-web-react/src/app/identity/group/GroupListItem.tsx @@ -1,12 +1,13 @@ +import { LockOutlined } from '@ant-design/icons'; import React from 'react'; import styled from 'styled-components'; -import { Button, List, message, Modal, Tag, Typography } from 'antd'; +import { List, Tag, Tooltip, Typography } from 'antd'; import { Link } from 'react-router-dom'; -import { DeleteOutlined } from '@ant-design/icons'; -import { CorpGroup, EntityType } from '../../../types.generated'; +import { CorpGroup, EntityType, OriginType } from '../../../types.generated'; import CustomAvatar from '../../shared/avatar/CustomAvatar'; import { useEntityRegistry } from '../../useEntityRegistry'; -import { useRemoveGroupMutation } from '../../../graphql/group.generated'; +import EntityDropdown from '../../entity/shared/EntityDropdown'; +import { EntityMenuItems } from '../../entity/shared/EntityDropdown/EntityDropdown'; type Props = { group: CorpGroup; @@ -27,40 +28,17 @@ const GroupHeaderContainer = styled.div` align-items: center; `; +const GroupItemButtonGroup = styled.div` + display: flex; + justify-content: space-evenly; + align-items: center; +`; + export default function GroupListItem({ group, onDelete }: Props) { const entityRegistry = useEntityRegistry(); const displayName = entityRegistry.getDisplayName(EntityType.CorpGroup, group); - - const [removeGroupMutation] = useRemoveGroupMutation(); - - const onRemoveGroup = async (urn: string) => { - try { - await removeGroupMutation({ - variables: { urn }, - }); - message.success({ content: 'Removed group.', duration: 2 }); - } catch (e: unknown) { - message.destroy(); - if (e instanceof Error) { - message.error({ content: `Failed to remove group: \n ${e.message || ''}`, duration: 3 }); - } - } - onDelete?.(); - }; - - const handleRemoveGroup = (urn: string) => { - Modal.confirm({ - title: `Confirm Group Removal`, - content: `Are you sure you want to remove this group?`, - onOk() { - onRemoveGroup(urn); - }, - onCancel() {}, - okText: 'Yes', - maskClosable: true, - closable: true, - }); - }; + const isExternalGroup: boolean = group.origin?.type === OriginType.External; + const externalGroupType: string = group.origin?.externalType || 'outside DataHub'; return ( @@ -79,11 +57,23 @@ export default function GroupListItem({ group, onDelete }: Props) { {(group as any).memberCount?.total || 0} members -
- -
+ + {isExternalGroup && ( + + + + )} + + ); diff --git a/datahub-web-react/src/app/identity/user/UserList.tsx b/datahub-web-react/src/app/identity/user/UserList.tsx index 43d400bfbea998..9e7b19cef66882 100644 --- a/datahub-web-react/src/app/identity/user/UserList.tsx +++ b/datahub-web-react/src/app/identity/user/UserList.tsx @@ -1,7 +1,8 @@ import React, { useEffect, useState } from 'react'; -import { Empty, List, message, Pagination } from 'antd'; +import { Button, Empty, List, message, Pagination } from 'antd'; import styled from 'styled-components'; import * as QueryString from 'query-string'; +import { UsergroupAddOutlined } from '@ant-design/icons'; import { useLocation } from 'react-router'; import UserListItem from './UserListItem'; import { Message } from '../../shared/Message'; @@ -10,6 +11,8 @@ import { CorpUser } from '../../../types.generated'; import TabToolbar from '../../entity/shared/components/styled/TabToolbar'; import { SearchBar } from '../../search/SearchBar'; import { useEntityRegistry } from '../../useEntityRegistry'; +import ViewInviteTokenModal from './ViewInviteTokenModal'; +import { useGetAuthenticatedUser } from '../../useGetAuthenticatedUser'; const UserContainer = styled.div``; @@ -36,8 +39,12 @@ export const UserList = () => { useEffect(() => setQuery(paramsQuery), [paramsQuery]); const [page, setPage] = useState(1); + const [isViewingInviteToken, setIsViewingInviteToken] = useState(false); const [removedUrns, setRemovedUrns] = useState([]); + const authenticatedUser = useGetAuthenticatedUser(); + const canManageUserCredentials = authenticatedUser?.platformPrivileges.manageUserCredentials || false; + const pageSize = DEFAULT_PAGE_SIZE; const start = (page - 1) * pageSize; @@ -76,7 +83,13 @@ export const UserList = () => {
- <> +
{ }} dataSource={filteredUsers} renderItem={(item: any) => ( - handleDelete(item.urn as string)} user={item as CorpUser} /> + handleDelete(item.urn as string)} + user={item as CorpUser} + canManageUserCredentials={canManageUserCredentials} + /> )} /> @@ -116,6 +133,12 @@ export const UserList = () => { showSizeChanger={false} /> + {canManageUserCredentials && ( + setIsViewingInviteToken(false)} + /> + )} ); diff --git a/datahub-web-react/src/app/identity/user/UserListItem.tsx b/datahub-web-react/src/app/identity/user/UserListItem.tsx index 3e7d3a88ee6818..035c49d08783a8 100644 --- a/datahub-web-react/src/app/identity/user/UserListItem.tsx +++ b/datahub-web-react/src/app/identity/user/UserListItem.tsx @@ -1,16 +1,18 @@ -import React from 'react'; +import React, { useState } from 'react'; import styled from 'styled-components'; -import { Button, List, message, Modal, Tag, Tooltip, Typography } from 'antd'; +import { Dropdown, List, Menu, Tag, Tooltip, Typography } from 'antd'; import { Link } from 'react-router-dom'; -import { DeleteOutlined } from '@ant-design/icons'; +import { DeleteOutlined, MoreOutlined, UnlockOutlined } from '@ant-design/icons'; import { CorpUser, CorpUserStatus, EntityType } from '../../../types.generated'; import CustomAvatar from '../../shared/avatar/CustomAvatar'; import { useEntityRegistry } from '../../useEntityRegistry'; -import { useRemoveUserMutation } from '../../../graphql/user.generated'; import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants'; +import ViewResetTokenModal from './ViewResetTokenModal'; +import useDeleteEntity from '../../entity/shared/EntityDropdown/useDeleteEntity'; type Props = { user: CorpUser; + canManageUserCredentials: boolean; onDelete?: () => void; }; @@ -34,40 +36,23 @@ const ButtonGroup = styled.div` align-items: center; `; -export default function UserListItem({ user, onDelete }: Props) { +const MenuIcon = styled(MoreOutlined)<{ fontSize?: number }>` + display: flex; + justify-content: center; + align-items: center; + font-size: ${(props) => props.fontSize || '24'}px; + height: 32px; + margin-left: 5px; +`; + +export default function UserListItem({ user, canManageUserCredentials, onDelete }: Props) { const entityRegistry = useEntityRegistry(); + const [isViewingResetToken, setIsViewingResetToken] = useState(false); const displayName = entityRegistry.getDisplayName(EntityType.CorpUser, user); + const isNativeUser: boolean = user.isNativeUser as boolean; + const shouldShowPasswordReset: boolean = canManageUserCredentials && isNativeUser; - const [removeUserMutation] = useRemoveUserMutation(); - - const onRemoveUser = async (urn: string) => { - try { - await removeUserMutation({ - variables: { urn }, - }); - message.success({ content: 'Removed user.', duration: 2 }); - } catch (e: unknown) { - message.destroy(); - if (e instanceof Error) { - message.error({ content: `Failed to remove user: \n ${e.message || ''}`, duration: 3 }); - } - } - onDelete?.(); - }; - - const handleRemoveUser = (urn: string) => { - Modal.confirm({ - title: `Confirm User Removal`, - content: `Are you sure you want to remove this user? Note that if you have SSO auto provisioning enabled, this user will be created when they log in again.`, - onOk() { - onRemoveUser(urn); - }, - onCancel() {}, - okText: 'Yes', - maskClosable: true, - closable: true, - }); - }; + const { onDeleteEntity } = useDeleteEntity(user.urn, EntityType.CorpUser, user, onDelete); const getUserStatusToolTip = (userStatus: CorpUserStatus) => { switch (userStatus) { @@ -118,10 +103,28 @@ export default function UserListItem({ user, onDelete }: Props) { - + + setIsViewingResetToken(true)}> +   Reset user password + + +  Delete + + + } + > + + + setIsViewingResetToken(false)} + /> ); } diff --git a/datahub-web-react/src/app/identity/user/ViewInviteTokenModal.tsx b/datahub-web-react/src/app/identity/user/ViewInviteTokenModal.tsx new file mode 100644 index 00000000000000..d8d2d9ccc38ecb --- /dev/null +++ b/datahub-web-react/src/app/identity/user/ViewInviteTokenModal.tsx @@ -0,0 +1,88 @@ +import { RedoOutlined } from '@ant-design/icons'; +import { Button, Modal, Typography } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import { PageRoutes } from '../../../conf/Global'; +import { + useCreateNativeUserInviteTokenMutation, + useGetNativeUserInviteTokenQuery, +} from '../../../graphql/user.generated'; + +const ModalSection = styled.div` + display: flex; + flex-direction: column; + padding-bottom: 12px; +`; + +const ModalSectionHeader = styled(Typography.Text)` + &&&& { + padding: 0px; + margin: 0px; + margin-bottom: 4px; + } +`; + +const ModalSectionParagraph = styled(Typography.Paragraph)` + &&&& { + padding: 0px; + margin: 0px; + } +`; + +const CreateInviteTokenButton = styled(Button)` + display: inline-block; + width: 20px; + margin-left: -6px; +`; + +type Props = { + visible: boolean; + onClose: () => void; +}; + +export default function ViewInviteTokenModal({ visible, onClose }: Props) { + const baseUrl = window.location.origin; + const { data: getNativeUserInviteTokenData } = useGetNativeUserInviteTokenQuery({ skip: !visible }); + + const [createNativeUserInviteToken, { data: createNativeUserInviteTokenData }] = + useCreateNativeUserInviteTokenMutation({}); + + const inviteToken = createNativeUserInviteTokenData?.createNativeUserInviteToken?.inviteToken + ? createNativeUserInviteTokenData?.createNativeUserInviteToken.inviteToken + : getNativeUserInviteTokenData?.getNativeUserInviteToken?.inviteToken || ''; + + const inviteLink = `${baseUrl}${PageRoutes.SIGN_UP}?invite_token=${inviteToken}`; + + return ( + + Invite new DataHub users + + } + visible={visible} + onCancel={onClose} + > + + Share invite link + + Share this invite link with other users in your workspace! + + + 
{inviteLink}
+ + + + Generate a new link + + Generate a new invite link! Note, any old links will cease to be active. + + createNativeUserInviteToken({})} size="small" type="text"> + + + + + ); +} diff --git a/datahub-web-react/src/app/identity/user/ViewResetTokenModal.tsx b/datahub-web-react/src/app/identity/user/ViewResetTokenModal.tsx new file mode 100644 index 00000000000000..e700a4d002d8d8 --- /dev/null +++ b/datahub-web-react/src/app/identity/user/ViewResetTokenModal.tsx @@ -0,0 +1,108 @@ +import { RedoOutlined } from '@ant-design/icons'; +import { Button, Modal, Typography } from 'antd'; +import React, { useState } from 'react'; +import styled from 'styled-components'; +import { PageRoutes } from '../../../conf/Global'; +import { useCreateNativeUserResetTokenMutation } from '../../../graphql/user.generated'; + +const ModalSection = styled.div` + display: flex; + flex-direction: column; + padding-bottom: 12px; +`; + +const ModalSectionHeader = styled(Typography.Text)` + &&&& { + padding: 0px; + margin: 0px; + margin-bottom: 4px; + } +`; + +const ModalSectionParagraph = styled(Typography.Paragraph)` + &&&& { + padding: 0px; + margin: 0px; + } +`; + +const CreateResetTokenButton = styled(Button)` + display: inline-block; + width: 20px; + margin-left: -6px; +`; + +type Props = { + visible: boolean; + userUrn: string; + username: string; + onClose: () => void; +}; + +export default function ViewResetTokenModal({ visible, userUrn, username, onClose }: Props) { + const baseUrl = window.location.origin; + const [hasGeneratedResetToken, setHasGeneratedResetToken] = useState(false); + + const [createNativeUserResetToken, { data: createNativeUserResetTokenData }] = + useCreateNativeUserResetTokenMutation({}); + + const resetToken = createNativeUserResetTokenData?.createNativeUserResetToken?.resetToken || ''; + + const inviteLink = `${baseUrl}${PageRoutes.RESET_CREDENTIALS}?reset_token=${resetToken}`; + + return ( + + Reset User Password + + } + visible={visible} + onCancel={onClose} + > + {hasGeneratedResetToken ? ( + + Share reset link + + Share this reset link to reset the credentials for {username}. + This link will expire in 24 hours. + + + 
{inviteLink}
+ + + ) : ( + + A new link must be generated + + You cannot view any old reset links. Please generate a new one below. + + + )} + + Generate a new link + + Generate a new reset link! Note, any old links will cease to be active. + + { + createNativeUserResetToken({ + variables: { + input: { + userUrn, + }, + }, + }); + setHasGeneratedResetToken(true); + }} + size="small" + type="text" + > + + + + + ); +} diff --git a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx index 78942ad9bded40..b67859a074c673 100644 --- a/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx +++ b/datahub-web-react/src/app/ingest/ManageIngestionPage.tsx @@ -1,7 +1,6 @@ import { Tabs, Typography } from 'antd'; import React, { useState } from 'react'; import styled from 'styled-components'; -import { SearchablePage } from '../search/SearchablePage'; import { IngestionSourceList } from './source/IngestionSourceList'; import { SecretsList } from './secret/SecretsList'; @@ -51,22 +50,18 @@ export const ManageIngestionPage = () => { }; return ( - - - - Manage Ingestion - - Create, schedule, and run DataHub ingestion pipelines. - - - onClickTab(tab)}> - - - - - {selectedTab === TabType.Sources ? : } - - - + + + Manage Ingestion + + Create, schedule, and run DataHub ingestion pipelines. + + + onClickTab(tab)}> + + + + {selectedTab === TabType.Sources ? : } + ); }; diff --git a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx index 7d76f3c74c1478..e204f93c6af07a 100644 --- a/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx +++ b/datahub-web-react/src/app/ingest/secret/SecretBuilderModal.tsx @@ -47,6 +47,7 @@ export const SecretBuilderModal = ({ initialState, visible, onSubmit, onCancel } title={Create a new Secret} visible={visible} onCancel={onCancel} + zIndex={1051} // one higher than other modals - needed for managed ingestion forms footer={ <> } + style={modalStyle} + bodyStyle={modalBodyStyle} + title={ + + Ingestion Run Details + + } visible={visible} onCancel={onClose} > {!data && loading && } {error && message.error('Failed to load execution details :(')}
- Captured Output - - 
{`${output}`}
- + + Status + {resultText} + {resultSummaryText} + + {result === SUCCESS && ( + + {data?.executionRequest?.id && } + + )} + + Logs + + + View logs that were collected during the ingestion run. + + + + + 
{`${logs}${!showExpandedLogs && '...'}`}
+ {!showExpandedLogs && ( + setShowExpandedLogs(true)}> + Show More + + )} + +
); diff --git a/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx b/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx new file mode 100644 index 00000000000000..7514e37c1c8bf7 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/IngestedAssets.tsx @@ -0,0 +1,135 @@ +import { Button, Typography } from 'antd'; +import React, { useState } from 'react'; +import styled from 'styled-components'; +import { useGetSearchResultsForMultipleQuery } from '../../../graphql/search.generated'; +import { EmbeddedListSearchModal } from '../../entity/shared/components/styled/search/EmbeddedListSearchModal'; +import { ANTD_GRAY } from '../../entity/shared/constants'; +import { formatNumber } from '../../shared/formatNumber'; +import { Message } from '../../shared/Message'; +import { extractEntityTypeCountsFromFacets } from './utils'; + +const HeaderContainer = styled.div` + display: flex; + justify-content: space-between; +`; + +const TitleContainer = styled.div``; + +const TotalContainer = styled.div` + display: flex; + flex-direction: column; + justify-content: right; + align-items: end; +`; + +const TotalText = styled(Typography.Text)` + font-size: 16px; + color: ${ANTD_GRAY[8]}; +`; + +const EntityCountsContainer = styled.div` + display: flex; + justify-content: left; + align-items: center; +`; + +const EntityCount = styled.div` + margin-right: 40px; + display: flex; + flex-direction: column; + align-items: flex-start; +`; + +const ViewAllButton = styled(Button)` + padding: 0px; + margin-top: 4px; +`; + +type Props = { + id: string; +}; + +export default function IngestedAssets({ id }: Props) { + // First thing to do is to search for all assets with the id as the run id! + const [showAssetSearch, setShowAssetSearch] = useState(false); + + // Execute search + const { data, loading, error } = useGetSearchResultsForMultipleQuery({ + variables: { + input: { + query: '*', + start: 0, + count: 1, + filters: [ + { + field: 'runId', + value: id, + }, + ], + }, + }, + }); + + // Parse filter values to get results. + const facets = data?.searchAcrossEntities?.facets; + + // Extract facets to construct the per-entity type breakdown stats + const hasEntityTypeFacet = (facets?.findIndex((facet) => facet.field === 'entity') || -1) >= 0; + const entityTypeFacets = + (hasEntityTypeFacet && facets?.filter((facet) => facet.field === 'entity')[0]) || undefined; + const hasSubTypeFacet = (facets?.findIndex((facet) => facet.field === 'typeNames') || -1) >= 0; + const subTypeFacets = (hasSubTypeFacet && facets?.filter((facet) => facet.field === 'typeNames')[0]) || undefined; + const countsByEntityType = + (entityTypeFacets && extractEntityTypeCountsFromFacets(entityTypeFacets, subTypeFacets)) || []; + + // The total number of assets ingested + const total = data?.searchAcrossEntities?.total || 0; + + return ( + <> + {error && } + + + Ingested Assets + {(loading && Loading...) || ( + <> + {(total > 0 && ( + + The following asset types were ingested during this run. + + )) || No assets were ingested.} + + )} + + {!loading && ( + + Total + + {formatNumber(total)} assets + + + )} + + + {countsByEntityType.map((entityCount) => ( + + + {formatNumber(entityCount.count)} + + {entityCount.displayName} + + ))} + + setShowAssetSearch(true)}> + View All + + {showAssetSearch && ( + setShowAssetSearch(false)} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceExecutionList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceExecutionList.tsx index 4fa839c95cc9db..dc4854a5100606 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceExecutionList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceExecutionList.tsx @@ -19,6 +19,17 @@ const ListContainer = styled.div` margin-left: 28px; `; +const StatusContainer = styled.div` + display: flex; + justify-content: left; + align-items: center; +`; + +const StatusButton = styled(Button)` + padding: 0px; + margin: 0px; +`; + type Props = { urn: string; lastRefresh: number; @@ -127,19 +138,19 @@ export const IngestionSourceExecutionList = ({ urn, lastRefresh, onRefresh }: Pr title: 'Status', dataIndex: 'status', key: 'status', - render: (status: any) => { + render: (status: any, record) => { const Icon = getExecutionRequestStatusIcon(status); const text = getExecutionRequestStatusDisplayText(status); const color = getExecutionRequestStatusDisplayColor(status); return ( - <> -
- {Icon && } + + {Icon && } + setFocusExecutionUrn(record.urn)}> {text || 'N/A'} -
- + + ); }, }, diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index 945f7f2eb7fd47..a13e40aabe2e88 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -28,6 +28,7 @@ import { UpdateIngestionSourceInput } from '../../../types.generated'; import { capitalizeFirstLetter } from '../../shared/textUtil'; import { SearchBar } from '../../search/SearchBar'; import { useEntityRegistry } from '../../useEntityRegistry'; +import { ExecutionDetailsModal } from './ExecutionRequestDetailsModal'; const SourceContainer = styled.div``; @@ -50,6 +51,11 @@ const StatusContainer = styled.div` align-items: center; `; +const StatusButton = styled(Button)` + padding: 0px; + margin: 0px; +`; + const ActionButtonContainer = styled.div` display: flex; justify-content: right; @@ -84,6 +90,7 @@ export const IngestionSourceList = () => { const [isBuildingSource, setIsBuildingSource] = useState(false); const [focusSourceUrn, setFocusSourceUrn] = useState(undefined); + const [focusExecutionUrn, setFocusExecutionUrn] = useState(undefined); const [lastRefresh, setLastRefresh] = useState(0); // Set of removed urns used to account for eventual consistency const [removedUrns, setRemovedUrns] = useState([]); @@ -111,13 +118,47 @@ export const IngestionSourceList = () => { const focusSource = (focusSourceUrn && filteredSources.find((source) => source.urn === focusSourceUrn)) || undefined; + const onRefresh = () => { + refetch(); + // Used to force a re-render of the child execution request list. + setLastRefresh(new Date().getMilliseconds()); + }; + + const executeIngestionSource = (urn: string) => { + createExecutionRequestMutation({ + variables: { + input: { + ingestionSourceUrn: urn, + }, + }, + }) + .then(() => { + message.success({ + content: `Successfully submitted ingestion execution request!`, + duration: 3, + }); + setInterval(() => onRefresh(), 3000); + }) + .catch((e) => { + message.destroy(); + message.error({ + content: `Failed to submit ingestion execution request!: \n ${e.message || ''}`, + duration: 3, + }); + }); + }; + const onCreateOrUpdateIngestionSourceSuccess = () => { setTimeout(() => refetch(), 2000); setIsBuildingSource(false); setFocusSourceUrn(undefined); }; - const createOrUpdateIngestionSource = (input: UpdateIngestionSourceInput, resetState: () => void) => { + const createOrUpdateIngestionSource = ( + input: UpdateIngestionSourceInput, + resetState: () => void, + shouldRun?: boolean, + ) => { if (focusSourceUrn) { // Update: updateIngestionSource({ variables: { urn: focusSourceUrn as string, input } }) @@ -128,6 +169,7 @@ export const IngestionSourceList = () => { }); onCreateOrUpdateIngestionSourceSuccess(); resetState(); + if (shouldRun) executeIngestionSource(focusSourceUrn); }) .catch((e) => { message.destroy(); @@ -139,15 +181,21 @@ export const IngestionSourceList = () => { } else { // Create createIngestionSource({ variables: { input } }) - .then(() => { - setTimeout(() => refetch(), 2000); + .then((result) => { + message.loading({ content: 'Loading...', duration: 2 }); + setTimeout(() => { + refetch(); + message.success({ + content: `Successfully created ingestion source!`, + duration: 3, + }); + if (shouldRun && result.data?.createIngestionSource) { + executeIngestionSource(result.data.createIngestionSource); + } + }, 2000); setIsBuildingSource(false); setFocusSourceUrn(undefined); resetState(); - message.success({ - content: `Successfully created ingestion source!`, - duration: 3, - }); // onCreateOrUpdateIngestionSourceSuccess(); }) .catch((e) => { @@ -164,36 +212,6 @@ export const IngestionSourceList = () => { setPage(newPage); }; - const onRefresh = () => { - refetch(); - // Used to force a re-render of the child execution request list. - setLastRefresh(new Date().getMilliseconds()); - }; - - const executeIngestionSource = (urn: string) => { - createExecutionRequestMutation({ - variables: { - input: { - ingestionSourceUrn: urn, - }, - }, - }) - .then(() => { - message.success({ - content: `Successfully submitted ingestion execution request!`, - duration: 3, - }); - setInterval(() => onRefresh(), 3000); - }) - .catch((e) => { - message.destroy(); - message.error({ - content: `Failed to submit ingestion execution request!: \n ${e.message || ''}`, - duration: 3, - }); - }); - }; - const deleteIngestionSource = async (urn: string) => { removeIngestionSourceMutation({ variables: { urn }, @@ -214,7 +232,7 @@ export const IngestionSourceList = () => { }); }; - const onSubmit = (recipeBuilderState: SourceBuilderState, resetState: () => void) => { + const onSubmit = (recipeBuilderState: SourceBuilderState, resetState: () => void, shouldRun?: boolean) => { createOrUpdateIngestionSource( { type: recipeBuilderState.type as string, @@ -236,6 +254,7 @@ export const IngestionSourceList = () => { }, }, resetState, + shouldRun, ); }; @@ -336,16 +355,18 @@ export const IngestionSourceList = () => { title: 'Last Status', dataIndex: 'lastExecStatus', key: 'lastExecStatus', - render: (status: any) => { + render: (status: any, record) => { const Icon = getExecutionRequestStatusIcon(status); const text = getExecutionRequestStatusDisplayText(status); const color = getExecutionRequestStatusDisplayColor(status); return ( {Icon && } - - {text || 'N/A'} - + setFocusExecutionUrn(record.lastExecUrn)}> + + {text || 'N/A'} + + ); }, @@ -393,6 +414,8 @@ export const IngestionSourceList = () => { schedule: source.schedule?.interval, timezone: source.schedule?.timezone, execCount: source.executions?.total || 0, + lastExecUrn: + source.executions?.total && source.executions?.total > 0 && source.executions?.executionRequests[0].urn, lastExecTime: source.executions?.total && source.executions?.total > 0 && @@ -478,6 +501,13 @@ export const IngestionSourceList = () => { onSubmit={onSubmit} onCancel={onCancel} /> + {focusExecutionUrn && ( + setFocusExecutionUrn(undefined)} + /> + )} ); }; diff --git a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx index 1ad28d8b92260f..2d755d29c9c8a8 100644 --- a/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/CreateScheduleStep.tsx @@ -1,5 +1,7 @@ -import { Button, Form, Input, Typography } from 'antd'; +import { Button, Form, Typography } from 'antd'; import React, { useMemo } from 'react'; +import { Cron } from 'react-js-cron'; +import 'react-js-cron/dist/styles.css'; import styled from 'styled-components'; import cronstrue from 'cronstrue'; import { CheckCircleOutlined } from '@ant-design/icons'; @@ -24,7 +26,6 @@ const SelectTemplateHeader = styled(Typography.Title)` const CronText = styled(Typography.Paragraph)` &&& { - margin-top: 8px; margin-bottom: 0px; } color: ${ANTD_GRAY[7]}; @@ -41,10 +42,21 @@ const ControlsContainer = styled.div` margin-top: 8px; `; +const StyledFormItem = styled(Form.Item)` + .cron-builder { + color: ${ANTD_GRAY[7]}; + } + .cron-builder-select { + min-width: 100px; + } +`; + const ItemDescriptionText = styled(Typography.Paragraph)``; +const DAILY_MIDNIGHT_CRON_INTERVAL = '0 0 * * *'; + export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps) => { - const interval = state.schedule?.interval || ''; + const interval = state.schedule?.interval?.replaceAll(', ', ' ') || DAILY_MIDNIGHT_CRON_INTERVAL; const timezone = state.schedule?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone; const setTimezone = (tz: string) => { @@ -110,9 +122,14 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps Configure your ingestion source to run on a schedule. - Schedule}> - Provide a custom cron schedule. - setCronInterval(e.target.value)} placeholder="* * * * *" /> + Schedule}> + {cronAsText.error && <>Invalid cron schedule. Cron must be of UNIX form:} {!cronAsText.text && ( @@ -127,7 +144,7 @@ export const CreateScheduleStep = ({ state, updateState, goTo, prev }: StepProps )} - + Timezone}> Select the timezone to run the cron schedule in. diff --git a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx index 55f673771a102b..4c75751f113b50 100644 --- a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx @@ -1,11 +1,16 @@ -import { Button, message, Typography } from 'antd'; +import { Alert, Button, Space, Typography } from 'antd'; import React, { useEffect, useState } from 'react'; import styled from 'styled-components'; import { StepProps } from './types'; -import { getSourceConfigs, jsonToYaml, yamlToJson } from '../utils'; +import { getSourceConfigs, jsonToYaml } from '../utils'; import { YamlEditor } from './YamlEditor'; import { ANTD_GRAY } from '../../../entity/shared/constants'; import { IngestionSourceBuilderStep } from './steps'; +import RecipeBuilder from './RecipeBuilder'; +import { CONNECTORS_WITH_FORM } from './RecipeForm/utils'; +import { getRecipeJson } from './RecipeForm/TestConnection/TestConnectionButton'; + +const LOOKML_DOC_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/looker#module-lookml'; const Section = styled.div` display: flex; @@ -35,36 +40,37 @@ const ControlsContainer = styled.div` export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) => { const existingRecipeJson = state.config?.recipe; const existingRecipeYaml = existingRecipeJson && jsonToYaml(existingRecipeJson); + const { type } = state; + const sourceConfigs = getSourceConfigs(type as string); - const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || ''); + const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || sourceConfigs.placeholderRecipe); useEffect(() => { - setStagedRecipeYml(existingRecipeYaml || ''); + if (existingRecipeYaml) { + setStagedRecipeYml(existingRecipeYaml); + } }, [existingRecipeYaml]); const [stepComplete, setStepComplete] = useState(false); - const { type } = state; - const sourceConfigs = getSourceConfigs(type as string); + const isEditing: boolean = prev === undefined; const displayRecipe = stagedRecipeYml || sourceConfigs.placeholderRecipe; const sourceDisplayName = sourceConfigs.displayName; const sourceDocumentationUrl = sourceConfigs.docsUrl; // Maybe undefined (in case of "custom") + // TODO: Delete LookML banner specific code + const isSourceLooker: boolean = sourceConfigs.type === 'looker'; + const [showLookerBanner, setShowLookerBanner] = useState(isSourceLooker && !isEditing); + useEffect(() => { - if (stagedRecipeYml && stagedRecipeYml.length > 0) { + if (stagedRecipeYml && stagedRecipeYml.length > 0 && !showLookerBanner) { setStepComplete(true); } - }, [stagedRecipeYml]); + }, [stagedRecipeYml, showLookerBanner]); const onClickNext = () => { - // Convert the recipe into it's json representation, and catch + report exceptions while we do it. - let recipeJson; - try { - recipeJson = yamlToJson(stagedRecipeYml); - } catch (e) { - message.warn('Found invalid YAML. Please check your recipe configuration.'); - return; - } + const recipeJson = getRecipeJson(stagedRecipeYml); + if (!recipeJson) return; const newState = { ...state, @@ -78,11 +84,59 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) goTo(IngestionSourceBuilderStep.CREATE_SCHEDULE); }; + if (type && CONNECTORS_WITH_FORM.has(type)) { + return ( + + ); + } + return ( <>
Configure {sourceDisplayName} Recipe + {showLookerBanner && ( + + + + You must acknowledge this message to proceed! + + +
+
+ To get complete Looker metadata integration (including Looker views and lineage to the + underlying warehouse tables), you must also use the{' '} + + DataHub lookml module + + . +
+
+ LookML ingestion cannot currently be performed via UI-based ingestion. This is a + known problem the DataHub team is working to solve! +
+ + + + + } + afterClose={() => setShowLookerBanner(false)} + /> + )} + {showLookerBanner &&
} For more information about how to configure a recipe, see the{' '} {sourceDisplayName} source docs. @@ -93,7 +147,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) - - +
+ 0)} + onClick={() => onClickCreate(false)} + > + Save + + +
); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx new file mode 100644 index 00000000000000..daf8b9bbed8085 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx @@ -0,0 +1,103 @@ +import { Button, message } from 'antd'; +import React, { useState } from 'react'; +import YAML from 'yamljs'; +import { CodeOutlined, FormOutlined } from '@ant-design/icons'; +import styled from 'styled-components/macro'; +import { ANTD_GRAY } from '../../../entity/shared/constants'; +import { YamlEditor } from './YamlEditor'; +import RecipeForm from './RecipeForm/RecipeForm'; + +export const ControlsContainer = styled.div` + display: flex; + justify-content: space-between; + margin-top: 8px; +`; + +const BorderedSection = styled.div` + display: flex; + flex-direction: column; + padding-bottom: 16px; + border: solid ${ANTD_GRAY[4]} 0.5px; +`; + +const StyledButton = styled(Button)<{ isSelected: boolean }>` + ${(props) => + props.isSelected && + ` + color: #1890ff; + &:focus { + color: #1890ff; + } + `} +`; + +const ButtonsWrapper = styled.div` + display: flex; + justify-content: flex-end; + margin-bottom: 10px; +`; + +interface Props { + type: string; + isEditing: boolean; + displayRecipe: string; + setStagedRecipe: (recipe: string) => void; + onClickNext: () => void; + goToPrevious?: () => void; +} + +function RecipeBuilder(props: Props) { + const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props; + + const [isViewingForm, setIsViewingForm] = useState(true); + + function switchViews(isFormView: boolean) { + try { + YAML.parse(displayRecipe); + setIsViewingForm(isFormView); + } catch (e) { + const messageText = (e as any).parsedLine + ? `Fix line ${(e as any).parsedLine} in your recipe` + : 'Please fix your recipe'; + message.warn(`Found invalid YAML. ${messageText} in order to switch views.`); + } + } + + return ( +
+ + switchViews(true)}> + Form + + switchViews(false)}> + YAML + + + {isViewingForm && ( + + )} + {!isViewingForm && ( + <> + + + + + + + + + )} +
+ ); +} + +export default RecipeBuilder; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx new file mode 100644 index 00000000000000..d6f130c31d3c4a --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx @@ -0,0 +1,151 @@ +import React from 'react'; +import { Button, Checkbox, Form, Input, Select, Tooltip } from 'antd'; +import styled from 'styled-components/macro'; +import { MinusCircleOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons'; +import { FieldType, RecipeField } from './utils'; +import { ANTD_GRAY } from '../../../../entity/shared/constants'; +import { Secret } from '../../../../../types.generated'; +import SecretField from './SecretField/SecretField'; + +const Label = styled.div` + font-weight: bold; + padding-bottom: 8px; +`; + +const StyledButton = styled(Button)` + color: ${ANTD_GRAY[7]}; + width: 80%; +`; + +const StyledQuestion = styled(QuestionCircleOutlined)` + color: rgba(0, 0, 0, 0.45); + margin-left: 4px; +`; + +const StyledRemoveIcon = styled(MinusCircleOutlined)` + font-size: 14px; + margin-left: 10px; +`; + +const StyledSelectField = styled(Select)` + margin-left: 10px; +`; + +const StyledFormItem = styled(Form.Item)<{ alignLeft: boolean; removeMargin: boolean }>` + margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')}; + + ${(props) => + props.alignLeft && + ` + .ant-form-item { + flex-direction: row; + + } + + .ant-form-item-label { + padding: 0; + margin-right: 10px; + } + `} +`; + +const ListWrapper = styled.div<{ removeMargin: boolean }>` + margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')}; +`; + +interface ListFieldProps { + field: RecipeField; + removeMargin?: boolean; +} + +interface SelectFieldProps { + field: RecipeField; +} + +function ListField({ field, removeMargin }: ListFieldProps) { + return ( + + {(fields, { add, remove }) => ( + + + {fields.map((item) => ( + + + + + remove(item.name)} /> + + ))} + add()} style={{ width: '80%' }} icon={}> + {field.buttonLabel} + + + )} + + ); +} + +function SelectField({ field }: SelectFieldProps) { + return ( + + {field.options && ( + + {field.options.map((option) => ( + {option.label} + ))} + + )} + + ); +} + +interface Props { + field: RecipeField; + secrets: Secret[]; + refetchSecrets: () => void; + removeMargin?: boolean; +} + +function FormField(props: Props) { + const { field, secrets, refetchSecrets, removeMargin } = props; + + if (field.type === FieldType.LIST) return ; + + if (field.type === FieldType.SELECT) return ; + + if (field.type === FieldType.SECRET) + return ; + + const isBoolean = field.type === FieldType.BOOLEAN; + const input = isBoolean ? : ; + const valuePropName = isBoolean ? 'checked' : 'value'; + const getValueFromEvent = isBoolean ? undefined : (e) => (e.target.value === '' ? null : e.target.value); + + return ( + + {input} + + ); +} + +export default FormField; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx new file mode 100644 index 00000000000000..d20ea3ba0b5073 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx @@ -0,0 +1,193 @@ +import { Button, Collapse, Form, message, Typography } from 'antd'; +import React from 'react'; +import { get } from 'lodash'; +import YAML from 'yamljs'; +import { ApiOutlined, FilterOutlined, SettingOutlined } from '@ant-design/icons'; +import styled from 'styled-components/macro'; +import { jsonToYaml } from '../../utils'; +import { RecipeField, RECIPE_FIELDS, setFieldValueOnRecipe } from './utils'; +import FormField from './FormField'; +import TestConnectionButton from './TestConnection/TestConnectionButton'; +import { SNOWFLAKE } from '../../conf/snowflake/snowflake'; +import { useListSecretsQuery } from '../../../../../graphql/ingestion.generated'; + +export const ControlsContainer = styled.div` + display: flex; + justify-content: space-between; + margin-top: 12px; +`; + +const StyledCollapse = styled(Collapse)` + margin-bottom: 16px; + + .ant-collapse-header { + font-size: 14px; + font-weight: bold; + padding: 12px 0; + } +`; + +const HeaderTitle = styled.span` + margin-left: 8px; +`; + +const MarginWrapper = styled.div` + margin-left: 20px; +`; + +const TestConnectionWrapper = styled.div` + display: flex; + justify-content: flex-end; + margin-top: 16px; +`; + +function getInitialValues(displayRecipe: string, allFields: any[]) { + const initialValues = {}; + let recipeObj; + try { + recipeObj = YAML.parse(displayRecipe); + } catch (e) { + message.warn('Found invalid YAML. Please check your recipe configuration.'); + return {}; + } + if (recipeObj) { + allFields.forEach((field) => { + initialValues[field.name] = + field.getValueFromRecipeOverride?.(recipeObj) || get(recipeObj, field.fieldPath); + }); + } + + return initialValues; +} + +function SectionHeader({ icon, text }: { icon: any; text: string }) { + return ( + + {icon} + {text} + + ); +} + +function shouldRenderFilterSectionHeader(field: RecipeField, index: number, filterFields: RecipeField[]) { + if (index === 0 && field.section) return true; + if (field.section && filterFields[index - 1].section !== field.section) return true; + return false; +} + +interface Props { + type: string; + isEditing: boolean; + displayRecipe: string; + setStagedRecipe: (recipe: string) => void; + onClickNext: () => void; + goToPrevious?: () => void; +} + +function RecipeForm(props: Props) { + const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props; + const { fields, advancedFields, filterFields } = RECIPE_FIELDS[type]; + const allFields = [...fields, ...advancedFields, ...filterFields]; + const { data, refetch: refetchSecrets } = useListSecretsQuery({ + variables: { + input: { + start: 0, + count: 1000, // get all secrets + }, + }, + }); + const secrets = + data?.listSecrets?.secrets.sort((secretA, secretB) => secretA.name.localeCompare(secretB.name)) || []; + + function updateFormValues(changedValues: any, allValues: any) { + let updatedValues = YAML.parse(displayRecipe); + + Object.keys(changedValues).forEach((fieldName) => { + const recipeField = allFields.find((f) => f.name === fieldName); + if (recipeField) { + updatedValues = + recipeField.setValueOnRecipeOverride?.(updatedValues, allValues[fieldName]) || + setFieldValueOnRecipe(updatedValues, allValues[fieldName], recipeField.fieldPath); + } + }); + + const stagedRecipe = jsonToYaml(JSON.stringify(updatedValues)); + setStagedRecipe(stagedRecipe); + } + + return ( +
+ + } text="Connection" />} key="0"> + {fields.map((field, i) => ( + + ))} + {type === SNOWFLAKE && ( + + + + )} + + + {filterFields.length > 0 && ( + + } text="Filter" />} + key="1" + > + {filterFields.map((field, i) => ( + <> + {shouldRenderFilterSectionHeader(field, i, filterFields) && ( + {field.section} + )} + + + + + ))} + + + )} + + } text="Advanced" />} + key="2" + > + {advancedFields.map((field, i) => ( + + ))} + + + + + + + + ); +} + +export default RecipeForm; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx new file mode 100644 index 00000000000000..01ca495900e400 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx @@ -0,0 +1,74 @@ +import React, { useState } from 'react'; +import { Button, message } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; +import { blue } from '@ant-design/colors'; +import styled from 'styled-components/macro'; +import { SecretBuilderModal } from '../../../../secret/SecretBuilderModal'; +import { useCreateSecretMutation } from '../../../../../../graphql/ingestion.generated'; +import { SecretBuilderState } from '../../../../secret/types'; + +const CreateButton = styled(Button)` + align-items: center; + display: flex; + justify-content: center; + margin: 8px 12px 4px 12px; + width: calc(100% - 24px); + + &:hover { + color: ${blue[5]}; + } + + .anticon-plus { + margin-right: 5px; + } +`; + +interface Props { + refetchSecrets: () => void; +} + +function CreateSecretButton({ refetchSecrets }: Props) { + const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); + const [createSecretMutation] = useCreateSecretMutation(); + + const createSecret = (state: SecretBuilderState, resetBuilderState: () => void) => { + createSecretMutation({ + variables: { + input: { + name: state.name as string, + value: state.value as string, + description: state.description as string, + }, + }, + }) + .then(() => { + setIsCreateModalVisible(false); + resetBuilderState(); + setTimeout(() => refetchSecrets(), 3000); + message.loading({ content: `Loading...`, duration: 3 }).then(() => { + message.success({ content: `Successfully created Secret!` }); + }); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to create secret: \n ${e.message || ''}` }); + }); + }; + + return ( + <> + setIsCreateModalVisible(true)} type="text"> + Create Secret + + {isCreateModalVisible && ( + setIsCreateModalVisible(false)} + onSubmit={createSecret} + /> + )} + + ); +} + +export default CreateSecretButton; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx new file mode 100644 index 00000000000000..3304dc1619eb5b --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx @@ -0,0 +1,42 @@ +import React from 'react'; +import { Divider, Form, Select } from 'antd'; +import styled from 'styled-components/macro'; +import { RecipeField } from '../utils'; +import { Secret } from '../../../../../../types.generated'; +import CreateSecretButton from './CreateSecretButton'; + +const StyledDivider = styled(Divider)` + margin: 0; +`; + +interface SecretFieldProps { + field: RecipeField; + secrets: Secret[]; + refetchSecrets: () => void; +} + +function SecretField({ field, secrets, refetchSecrets }: SecretFieldProps) { + return ( + + + + ); +} + +export default SecretField; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/ConnectionCapabilityView.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/ConnectionCapabilityView.tsx new file mode 100644 index 00000000000000..e2a9b0f146e405 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/ConnectionCapabilityView.tsx @@ -0,0 +1,74 @@ +import { CheckOutlined, CloseOutlined, QuestionCircleOutlined } from '@ant-design/icons'; +import { Tooltip } from 'antd'; +import React from 'react'; +import { green, red } from '@ant-design/colors'; +import styled from 'styled-components/macro'; +import { ANTD_GRAY } from '../../../../../entity/shared/constants'; + +const CapabilityWrapper = styled.div` + align-items: center; + display: flex; + margin: 10px 0; +`; + +const CapabilityName = styled.span` + color: ${ANTD_GRAY[8]}; + font-size: 18px; + margin-right: 12px; +`; + +const CapabilityMessage = styled.span<{ success: boolean }>` + color: ${(props) => (props.success ? `${green[6]}` : `${red[5]}`)}; + font-size: 12px; + flex: 1; + padding-left: 4px; +`; + +const StyledQuestion = styled(QuestionCircleOutlined)` + color: rgba(0, 0, 0, 0.45); + margin-left: 4px; +`; + +export const StyledCheck = styled(CheckOutlined)` + color: ${green[6]}; + margin-right: 15px; +`; + +export const StyledClose = styled(CloseOutlined)` + color: ${red[5]}; + margin-right: 15px; +`; + +const NumberWrapper = styled.span` + margin-right: 8px; +`; + +interface Props { + success: boolean; + capability: string; + displayMessage: string | null; + tooltipMessage: string | null; + number?: number; +} + +function ConnectionCapabilityView({ success, capability, displayMessage, tooltipMessage, number }: Props) { + return ( + + + {success ? : } + {number ? {number}. : ''} + {capability} + + + {displayMessage} + {tooltipMessage && ( + + + + )} + + + ); +} + +export default ConnectionCapabilityView; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionButton.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionButton.tsx new file mode 100644 index 00000000000000..0d6902ee620d05 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionButton.tsx @@ -0,0 +1,128 @@ +import { CheckCircleOutlined } from '@ant-design/icons'; +import { Button, message } from 'antd'; +import React, { useEffect, useState } from 'react'; +import { green } from '@ant-design/colors'; +import { + useCreateTestConnectionRequestMutation, + useGetIngestionExecutionRequestLazyQuery, +} from '../../../../../../graphql/ingestion.generated'; +import { FAILURE, getSourceConfigs, RUNNING, yamlToJson } from '../../../utils'; +import { TestConnectionResult } from './types'; +import TestConnectionModal from './TestConnectionModal'; + +export function getRecipeJson(recipeYaml: string) { + // Convert the recipe into it's json representation, and catch + report exceptions while we do it. + let recipeJson; + try { + recipeJson = yamlToJson(recipeYaml); + } catch (e) { + const messageText = (e as any).parsedLine + ? `Please fix line ${(e as any).parsedLine} in your recipe.` + : 'Please check your recipe configuration.'; + message.warn(`Found invalid YAML. ${messageText}`); + return null; + } + return recipeJson; +} + +interface Props { + type: string; + recipe: string; +} + +function TestConnectionButton(props: Props) { + const { type, recipe } = props; + const [isLoading, setIsLoading] = useState(false); + const [isModalVisible, setIsModalVisible] = useState(false); + const [pollingInterval, setPollingInterval] = useState(null); + const [testConnectionResult, setTestConnectionResult] = useState(null); + const [createTestConnectionRequest, { data: requestData }] = useCreateTestConnectionRequestMutation(); + const [getIngestionExecutionRequest, { data: resultData, loading }] = useGetIngestionExecutionRequestLazyQuery(); + + const sourceConfigs = getSourceConfigs(type); + + useEffect(() => { + if (requestData && requestData.createTestConnectionRequest) { + const interval = setInterval( + () => + getIngestionExecutionRequest({ + variables: { urn: requestData.createTestConnectionRequest as string }, + }), + 2000, + ); + setIsLoading(true); + setIsModalVisible(true); + setPollingInterval(interval); + } + }, [requestData, getIngestionExecutionRequest]); + + useEffect(() => { + if (!loading && resultData) { + const result = resultData.executionRequest?.result; + if (result && result.status !== RUNNING) { + if (result.status === FAILURE) { + message.error( + 'Something went wrong with your connection test. Please check your recipe and try again.', + ); + setIsModalVisible(false); + } + if (result.structuredReport) { + const testConnectionReport = JSON.parse(result.structuredReport.serializedValue); + setTestConnectionResult(testConnectionReport); + } + if (pollingInterval) clearInterval(pollingInterval); + setIsLoading(false); + } + } + }, [resultData, pollingInterval, loading]); + + useEffect(() => { + if (!isModalVisible && pollingInterval) { + clearInterval(pollingInterval); + } + }, [isModalVisible, pollingInterval]); + + function testConnection() { + const recipeJson = getRecipeJson(recipe); + if (recipeJson) { + createTestConnectionRequest({ variables: { input: { recipe: recipeJson } } }) + .then((res) => + getIngestionExecutionRequest({ + variables: { urn: res.data?.createTestConnectionRequest as string }, + }), + ) + .catch(() => { + message.error( + 'There was an unexpected error when trying to test your connection. Please try again.', + ); + }); + + setIsLoading(true); + setIsModalVisible(true); + } + } + + const internalFailure = !!testConnectionResult?.internal_failure; + const basicConnectivityFailure = testConnectionResult?.basic_connectivity?.capable === false; + const testConnectionFailed = internalFailure || basicConnectivityFailure; + + return ( + <> + + {isModalVisible && ( + setIsModalVisible(false)} + /> + )} + + ); +} + +export default TestConnectionButton; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx new file mode 100644 index 00000000000000..0b96c401857b6c --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx @@ -0,0 +1,191 @@ +import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; +import { Button, Divider, Modal, Typography } from 'antd'; +import React from 'react'; +import { green, red } from '@ant-design/colors'; +import styled from 'styled-components/macro'; +import { ReactComponent as LoadingSvg } from '../../../../../../images/datahub-logo-color-loading_pendulum.svg'; +import { ANTD_GRAY } from '../../../../../entity/shared/constants'; +import ConnectionCapabilityView from './ConnectionCapabilityView'; +import { SourceConfig } from '../../../conf/types'; +import { CapabilityReport, SourceCapability, TestConnectionResult } from './types'; + +const LoadingWrapper = styled.div` + display: flex; + justify-content: center; + margin: 50px 0 60px 0; +`; + +const LoadingSubheader = styled.div` + display: flex; + justify-content: center; + font-size: 12px; +`; + +const LoadingHeader = styled(Typography.Title)` + display: flex; + justify-content: center; +`; + +const ResultsHeader = styled.div<{ success: boolean }>` + align-items: center; + color: ${(props) => (props.success ? `${green[6]}` : `${red[5]}`)}; + display: flex; + margin-bottom: 5px; + font-size: 20px; + font-weight: 550; + + svg { + margin-right: 6px; + } +`; + +const ResultsSubHeader = styled.div` + color: ${ANTD_GRAY[7]}; +`; + +const ResultsWrapper = styled.div` + padding: 0 10px; +`; + +const ModalHeader = styled.div` + align-items: center; + display: flex; + padding: 10px 10px 0 10px; + padding: 5px; + font-size: 20px; +`; + +const SourceIcon = styled.img` + height: 22px; + width: 22px; + margin-right: 10px; +`; + +const CapabilitiesHeader = styled.div` + margin: -5px 0 20px 0; +`; + +const CapabilitiesTitle = styled.div` + font-size: 18px; + font-weight: bold; + margin-bottom: 5px; +`; + +const StyledCheck = styled(CheckOutlined)` + color: ${green[6]}; + margin-right: 5px; +`; + +const StyledClose = styled(CloseOutlined)` + color: ${red[5]}; + margin-right: 5px; +`; + +interface Props { + isLoading: boolean; + testConnectionFailed: boolean; + sourceConfig: SourceConfig; + testConnectionResult: TestConnectionResult | null; + hideModal: () => void; +} + +function TestConnectionModal({ + isLoading, + testConnectionFailed, + sourceConfig, + testConnectionResult, + hideModal, +}: Props) { + return ( + Done} + title={ + + + {sourceConfig.displayName} Connection Test + + } + width={750} + > + {isLoading && ( + + Testing your connection... + This could take a few minutes + + + + + )} + {!isLoading && ( + + + {testConnectionFailed ? ( + <> + Connection Failed + + ) : ( + <> + Connection Succeeded + + )} + + + {testConnectionFailed + ? `A connection was not able to be established with ${sourceConfig.displayName}.` + : `A connection was successfully established with ${sourceConfig.displayName}.`} + + + {testConnectionResult?.internal_failure ? ( + + ) : ( + + Capabilities + + The following connector capabilities are supported with your credentials + + + )} + {testConnectionResult?.basic_connectivity && ( + + )} + {testConnectionResult?.capability_report && + Object.keys(testConnectionResult.capability_report).map((capabilityKey, index) => { + return ( + + ); + })} + + )} + + ); +} + +export default TestConnectionModal; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/types.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/types.ts new file mode 100644 index 00000000000000..3395f0c67d8c8a --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/types.ts @@ -0,0 +1,32 @@ +export enum SourceCapability { + PLATFORM_INSTANCE = 'Platform Instance', + DOMAINS = 'Domains', + DATA_PROFILING = 'Data Profiling', + USAGE_STATS = 'Usage Stats', + PARTITION_SUPPORT = 'Partition Support', + DESCRIPTIONS = 'Descriptions', + LINEAGE_COARSE = 'Table-Level Lineage', + LINEAGE_FINE = 'Column-level Lineage', + OWNERSHIP = 'Extract Ownership', + DELETION_DETECTION = 'Detect Deleted Entities', + TAGS = 'Extract Tags', + SCHEMA_METADATA = 'Schema Metadata', + CONTAINERS = 'Asset Containers', +} + +export interface ConnectionCapability { + capable: boolean; + failure_reason: string | null; + mitigation_message: string | null; +} + +export interface CapabilityReport { + [key: string]: ConnectionCapability; +} + +export interface TestConnectionResult { + internal_failure?: boolean; + internal_failure_reason?: string; + basic_connectivity?: ConnectionCapability; + capability_report?: CapabilityReport; +} diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts new file mode 100644 index 00000000000000..64604ea350d283 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts @@ -0,0 +1,120 @@ +import { setFieldValueOnRecipe, setListValuesOnRecipe } from '../utils'; + +describe('setFieldValueOnRecipe', () => { + const accountIdFieldPath = 'source.config.account_id'; + const profilingEnabledFieldPath = 'source.config.profiling.enabled'; + + it('should set the field value on a recipe object when it was not defined', () => { + const recipe = { source: { config: {} } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, 'test', accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { account_id: 'test' } } }); + }); + + it('should update the field value on a recipe object when it was defined', () => { + const recipe = { source: { config: { account_id: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, 'edited!', accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { account_id: 'edited!' } } }); + }); + + it('should update the field value on a recipe without changing any other fields', () => { + const recipe = { source: { config: { test: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, 'edited!', accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', account_id: 'edited!' } } }); + }); + + it('should clear the key: value pair when passing in null', () => { + const recipe = { source: { config: { existingField: true, account_id: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, null, accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { existingField: true } } }); + }); + + it('should return the original recipe when passing in undefined', () => { + const recipe = { source: { config: { test: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, undefined, accountIdFieldPath); + expect(updatedRecipe).toMatchObject(recipe); + }); + + it('should set the field value on a recipe object when it was not defined and has a parent', () => { + const recipe = { source: { config: {} } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, true, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { profiling: { enabled: true } } } }); + }); + + it('should update the field value on a recipe object when it was defined and has a parent', () => { + const recipe = { source: { config: { profiling: { enabled: true } } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, false, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { profiling: { enabled: false } } } }); + }); + + it('should update the field value with a parent on a recipe without changing any other fields', () => { + const recipe = { source: { config: { test: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, false, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', profiling: { enabled: false } } } }); + }); + + it('should clear the field and its parent when passing in null and field is only child of parent', () => { + const recipe = { source: { config: { test: 'test', profiling: { enabled: true } } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, null, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test' } } }); + }); + + it('should clear the field but not its parent when passing in null and parent has other children', () => { + const recipe = { source: { config: { test: 'test', profiling: { enabled: true, testing: 'hello' } } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, null, 'source.config.profiling.testing'); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', profiling: { enabled: true } } } }); + }); +}); + +describe('setListValuesOnRecipe', () => { + const tableAllowFieldPath = 'source.config.table_pattern.allow'; + + it('should update list values on a recipe when it was not defined', () => { + const recipe = { source: { config: {} } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }); + }); + + it('should update list values on a recipe when it was defined', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern(edit)'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern(edit)'] } } }, + }); + }); + + it('should append list values on a recipe', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern', 'new'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern', 'new'] } } }, + }); + }); + + it('should remove list values on a recipe', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern', 'remove_me'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern'] } } }, + }); + }); + + it('should remove empty values from the list when updating a recipe', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern', '', '', ''], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern'] } } }, + }); + }); + + it('should clear the value and its parent when passing in empty list and parent has no other children', () => { + const recipe = { source: { config: { existingField: true, table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, [], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { existingField: true } } }); + }); + + it('should clear the value but not its parent when passing in empty list and parent has other children', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'], deny: ['test_deny'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, [], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { table_pattern: { deny: ['test_deny'] } } } }); + }); +}); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.tsx new file mode 100644 index 00000000000000..e6eed8605ecd39 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.tsx @@ -0,0 +1,639 @@ +import React from 'react'; +import { set, get } from 'lodash'; +import { SNOWFLAKE } from '../../conf/snowflake/snowflake'; +import { BIGQUERY } from '../../conf/bigquery/bigquery'; +import { REDSHIFT } from '../../conf/redshift/redshift'; +import { LOOKER } from '../../conf/looker/looker'; +import { TABLEAU } from '../../conf/tableau/tableau'; + +export enum FieldType { + TEXT, + BOOLEAN, + LIST, + SELECT, + SECRET, +} + +interface Option { + label: string; + value: string; +} + +export interface RecipeField { + name: string; + label: string; + tooltip: string | React.ReactNode; + type: FieldType; + fieldPath: string; + rules: any[] | null; + section?: string; + options?: Option[]; + buttonLabel?: string; + getValueFromRecipeOverride?: (recipe: any) => any; + setValueOnRecipeOverride?: (recipe: any, value: any) => any; +} + +function clearFieldAndParents(recipe: any, fieldPath: string) { + set(recipe, fieldPath, undefined); + + const updatedFieldPath = fieldPath.split('.').slice(0, -1).join('.'); // remove last item from fieldPath + if (updatedFieldPath) { + const parentKeys = Object.keys(get(recipe, updatedFieldPath)); + + // only child left is what we just set as undefined + if (parentKeys.length === 1) { + clearFieldAndParents(recipe, updatedFieldPath); + } + } + return recipe; +} + +export function setFieldValueOnRecipe(recipe: any, value: any, fieldPath: string) { + const updatedRecipe = { ...recipe }; + if (value !== undefined) { + if (value === null) { + clearFieldAndParents(updatedRecipe, fieldPath); + return updatedRecipe; + } + set(updatedRecipe, fieldPath, value); + } + return updatedRecipe; +} + +export function setListValuesOnRecipe(recipe: any, values: string[] | undefined, fieldPath: string) { + const updatedRecipe = { ...recipe }; + if (values !== undefined) { + const filteredValues: string[] | undefined = values.filter((v) => !!v); + return filteredValues.length + ? setFieldValueOnRecipe(updatedRecipe, filteredValues, fieldPath) + : setFieldValueOnRecipe(updatedRecipe, null, fieldPath); + } + return updatedRecipe; +} + +export const SNOWFLAKE_ACCOUNT_ID: RecipeField = { + name: 'account_id', + label: 'Account ID', + tooltip: 'Snowflake account. e.g. abc48144', + type: FieldType.TEXT, + fieldPath: 'source.config.account_id', + rules: null, +}; + +export const SNOWFLAKE_WAREHOUSE: RecipeField = { + name: 'warehouse', + label: 'Warehouse', + tooltip: 'Snowflake warehouse.', + type: FieldType.TEXT, + fieldPath: 'source.config.warehouse', + rules: null, +}; + +export const SNOWFLAKE_USERNAME: RecipeField = { + name: 'username', + label: 'Username', + tooltip: 'Snowflake username.', + type: FieldType.SECRET, + fieldPath: 'source.config.username', + rules: null, +}; + +export const SNOWFLAKE_PASSWORD: RecipeField = { + name: 'password', + label: 'Password', + tooltip: 'Snowflake password.', + type: FieldType.SECRET, + fieldPath: 'source.config.password', + rules: null, +}; + +export const SNOWFLAKE_ROLE: RecipeField = { + name: 'role', + label: 'Role', + tooltip: 'Snowflake role.', + type: FieldType.TEXT, + fieldPath: 'source.config.role', + rules: null, +}; + +export const BIGQUERY_PROJECT_ID: RecipeField = { + name: 'project_id', + label: 'BigQuery Project ID', + tooltip: 'Project ID where you have rights to run queries and create tables.', + type: FieldType.TEXT, + fieldPath: 'source.config.project_id', + rules: null, +}; + +export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = { + name: 'credential.project_id', + label: 'Credentials Project ID', + tooltip: 'Project id to set the credentials.', + type: FieldType.TEXT, + fieldPath: 'source.config.credential.project_id', + rules: null, +}; + +export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = { + name: 'credential.private_key_id', + label: 'Private Key Id', + tooltip: 'Private key id.', + type: FieldType.TEXT, + fieldPath: 'source.config.credential.private_key_id', + rules: null, +}; + +export const BIGQUERY_PRIVATE_KEY: RecipeField = { + name: 'credential.private_key', + label: 'Private Key', + tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".', + type: FieldType.TEXT, + fieldPath: 'source.config.credential.private_key', + rules: null, +}; + +export const BIGQUERY_CLIENT_EMAIL: RecipeField = { + name: 'credential.client_email', + label: 'Client Email', + tooltip: 'Client email.', + type: FieldType.TEXT, + fieldPath: 'source.config.credential.client_email', + rules: null, +}; + +export const BIGQUERY_CLIENT_ID: RecipeField = { + name: 'credential.client_id', + label: 'Client ID', + tooltip: 'Client ID.', + type: FieldType.TEXT, + fieldPath: 'source.config.credential.client_id', + rules: null, +}; + +export const REDSHIFT_HOST_PORT: RecipeField = { + name: 'host_port', + label: 'Host Port', + tooltip: 'Host URL.', + type: FieldType.TEXT, + fieldPath: 'source.config.host_port', + rules: null, +}; + +export const REDSHIFT_DATABASE: RecipeField = { + name: 'database', + label: 'Database', + tooltip: 'Database (catalog).', + type: FieldType.TEXT, + fieldPath: 'source.config.database', + rules: null, +}; + +export const REDSHIFT_USERNAME: RecipeField = { + name: 'redshift.username', + label: 'Redshift username', + tooltip: 'Username', + type: FieldType.TEXT, + fieldPath: 'source.config.username', + rules: null, +}; + +export const REDSHIFT_PASSWORD: RecipeField = { + name: 'redshift.password', + label: 'Redshift password', + tooltip: 'Password', + type: FieldType.TEXT, + fieldPath: 'source.config.password', + rules: null, +}; + +export const TABLEAU_CONNECTION_URI: RecipeField = { + name: 'connect_uri', + label: 'Connection URI', + tooltip: 'Tableau host URL.', + type: FieldType.TEXT, + fieldPath: 'source.config.connect_uri', + rules: null, +}; + +const tableauProjectFieldPath = 'source.config.projects'; +export const TABLEAU_PROJECT: RecipeField = { + name: 'projects', + label: 'Projects', + tooltip: 'List of projects', + type: FieldType.LIST, + buttonLabel: 'Add project', + fieldPath: tableauProjectFieldPath, + rules: null, + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableauProjectFieldPath), +}; + +export const TABLEAU_SITE: RecipeField = { + name: 'site', + label: 'Tableau Site', + tooltip: + 'Tableau Site. Always required for Tableau Online. Use empty string to connect with Default site on Tableau Server.', + type: FieldType.TEXT, + fieldPath: 'source.config.site', + rules: null, +}; + +export const TABLEAU_USERNAME: RecipeField = { + name: 'tableau.username', + label: 'Username', + tooltip: 'Tableau username, must be set if authenticating using username/password.', + type: FieldType.TEXT, + fieldPath: 'source.config.username', + rules: null, +}; + +export const TABLEAU_PASSWORD: RecipeField = { + name: 'tableau.password', + label: 'Password', + tooltip: 'Tableau password, must be set if authenticating using username/password.', + type: FieldType.TEXT, + fieldPath: 'source.config.password', + rules: null, +}; + +export const LOOKER_BASE_URL: RecipeField = { + name: 'base_url', + label: 'Base URL', + tooltip: + 'Url to your Looker instance: https://company.looker.com:19999 or https://looker.company.com, or similar.Used for making API calls to Looker and constructing clickable dashboard and chart urls.', + type: FieldType.TEXT, + fieldPath: 'source.config.base_url', + rules: null, +}; + +export const LOOKER_CLIENT_ID: RecipeField = { + name: 'client_id', + label: 'Client ID', + tooltip: 'Looker API client id.', + type: FieldType.TEXT, + fieldPath: 'source.config.client_id', + rules: null, +}; + +export const LOOKER_CLIENT_SECRET: RecipeField = { + name: 'client_secret', + label: 'Client Secret', + tooltip: 'Looker API client secret.', + type: FieldType.TEXT, + fieldPath: 'source.config.client_secret', + rules: null, +}; + +const includeLineageFieldPathA = 'source.config.include_table_lineage'; +const includeLineageFieldPathB = 'source.config.include_view_lineage'; +export const INCLUDE_LINEAGE: RecipeField = { + name: 'include_lineage', + label: 'Include Lineage', + tooltip: 'Include Table and View lineage in your ingestion.', + type: FieldType.BOOLEAN, + fieldPath: includeLineageFieldPathA, + rules: null, + getValueFromRecipeOverride: (recipe: any) => + get(recipe, includeLineageFieldPathA) && get(recipe, includeLineageFieldPathB), + setValueOnRecipeOverride: (recipe: any, value: boolean) => { + let updatedRecipe = setFieldValueOnRecipe(recipe, value, includeLineageFieldPathA); + updatedRecipe = setFieldValueOnRecipe(updatedRecipe, value, includeLineageFieldPathB); + return updatedRecipe; + }, +}; + +export const PROFILING_ENABLED: RecipeField = { + name: 'profiling.enabled', + label: 'Enable Profiling', + tooltip: 'Whether profiling should be done.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.profiling.enabled', + rules: null, +}; + +export const STATEFUL_INGESTION_ENABLED: RecipeField = { + name: 'stateful_ingestion.enabled', + label: 'Enable Stateful Ingestion', + tooltip: 'Enable the type of the ingestion state provider registered with datahub.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.stateful_ingestion.enabled', + rules: null, +}; + +export const UPSTREAM_LINEAGE_IN_REPORT: RecipeField = { + name: 'upstream_lineage_in_report', + label: 'Include Upstream Lineage In Report.', + tooltip: 'Remove stale datasets from datahub once they have been deleted in the source.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.upstream_lineage_in_report', + rules: null, +}; + +export const TABLE_LINEAGE_MODE: RecipeField = { + name: 'table_lineage_mode', + label: 'Table Lineage Mode', + tooltip: ( +
+

+ Which table lineage collector mode to use. Check out{' '} + + the documentation + {' '} + explaining the difference between the three available modes. +

+
+ ), + type: FieldType.SELECT, + fieldPath: 'source.config.table_lineage_mode', + rules: null, + options: [ + { label: 'stl_scan_based', value: 'stl_scan_based' }, + { label: 'sql_based', value: 'sql_based' }, + { label: 'mixed', value: 'mixed' }, + ], +}; + +export const INGEST_TAGS: RecipeField = { + name: 'ingest_tags', + label: 'Ingest Tags', + tooltip: 'Ingest Tags from source. This will override Tags entered from UI', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.ingest_tags', + rules: null, +}; + +export const INGEST_OWNER: RecipeField = { + name: 'ingest_owner', + label: 'Ingest Owner', + tooltip: 'Ingest Owner from source. This will override Owner info entered from UI', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.ingest_owner', + rules: null, +}; + +export const GITHUB_INFO_REPO: RecipeField = { + name: 'github_info.repo', + label: 'GitHub Repo', + tooltip: ( +
+

+ Name of your github repo. e.g. repo for{' '} + + https://github.com/datahub-project/datahub + {' '} + is datahub-project/datahub. +

+
+ ), + type: FieldType.TEXT, + fieldPath: 'source.config.github_info.repo', + rules: null, +}; + +export const EXTRACT_USAGE_HISTORY: RecipeField = { + name: 'extract_usage_history', + label: 'Extract Usage History', + tooltip: + 'Experimental (Subject to breaking change) -- Whether to ingest usage statistics for dashboards. Setting this to True will query looker system activity explores to fetch historical dashboard usage.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.extract_usage_history', + rules: null, +}; + +export const EXTRACT_OWNERS: RecipeField = { + name: 'extract_owners', + label: 'Extract Owners', + tooltip: + 'When enabled, extracts ownership from Looker directly. When disabled, ownership is left empty for dashboards and charts.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.extract_owners', + rules: null, +}; + +export const SKIP_PERSONAL_FOLDERS: RecipeField = { + name: 'skip_personal_folders', + label: 'Skip Personal Folders', + tooltip: + 'Whether to skip ingestion of dashboards in personal folders. Setting this to True will only ingest dashboards in the Shared folder space.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.skip_personal_folders', + rules: null, +}; + +const databaseAllowFieldPath = 'source.config.database_pattern.allow'; +export const DATABASE_ALLOW: RecipeField = { + name: 'database_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: databaseAllowFieldPath, + rules: null, + section: 'Databases', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, databaseAllowFieldPath), +}; + +const databaseDenyFieldPath = 'source.config.database_pattern.deny'; +export const DATABASE_DENY: RecipeField = { + name: 'database_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: databaseDenyFieldPath, + rules: null, + section: 'Databases', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, databaseDenyFieldPath), +}; + +const schemaAllowFieldPath = 'source.config.schema_pattern.allow'; +export const SCHEMA_ALLOW: RecipeField = { + name: 'schema_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: schemaAllowFieldPath, + rules: null, + section: 'Schemas', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, schemaAllowFieldPath), +}; + +const schemaDenyFieldPath = 'source.config.schema_pattern.deny'; +export const SCHEMA_DENY: RecipeField = { + name: 'schema_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: schemaDenyFieldPath, + rules: null, + section: 'Schemas', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, schemaDenyFieldPath), +}; + +const viewAllowFieldPath = 'source.config.view_pattern.allow'; +export const VIEW_ALLOW: RecipeField = { + name: 'view_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: viewAllowFieldPath, + rules: null, + section: 'Views', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, viewAllowFieldPath), +}; + +const viewDenyFieldPath = 'source.config.view_pattern.deny'; +export const VIEW_DENY: RecipeField = { + name: 'view_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: viewDenyFieldPath, + rules: null, + section: 'Views', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, viewDenyFieldPath), +}; + +const tableAllowFieldPath = 'source.config.table_pattern.allow'; +export const TABLE_ALLOW: RecipeField = { + name: 'table_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: tableAllowFieldPath, + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableAllowFieldPath), +}; + +const tableDenyFieldPath = 'source.config.table_pattern.deny'; +export const TABLE_DENY: RecipeField = { + name: 'table_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: tableDenyFieldPath, + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableDenyFieldPath), +}; + +const chartAllowFieldPath = 'source.config.chart_pattern.allow'; +export const CHART_ALLOW: RecipeField = { + name: 'chart_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: chartAllowFieldPath, + rules: null, + section: 'Charts', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, chartAllowFieldPath), +}; + +const chartDenyFieldPath = 'source.config.chart_pattern.deny'; +export const CHART_DENY: RecipeField = { + name: 'chart_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: chartDenyFieldPath, + rules: null, + section: 'Charts', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, chartDenyFieldPath), +}; + +const dashboardAllowFieldPath = 'source.config.dashboard_pattern.allow'; +export const DASHBOARD_ALLOW: RecipeField = { + name: 'dashboard_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: dashboardAllowFieldPath, + rules: null, + section: 'Dashboards', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, dashboardAllowFieldPath), +}; + +const dashboardDenyFieldPath = 'source.config.dashboard_pattern.deny'; +export const DASHBOARD_DENY: RecipeField = { + name: 'dashboard_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: dashboardDenyFieldPath, + rules: null, + section: 'Dashboards', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, dashboardDenyFieldPath), +}; + +export const RECIPE_FIELDS = { + [SNOWFLAKE]: { + fields: [SNOWFLAKE_ACCOUNT_ID, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_USERNAME, SNOWFLAKE_PASSWORD, SNOWFLAKE_ROLE], + advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED], + filterFields: [ + TABLE_ALLOW, + TABLE_DENY, + DATABASE_ALLOW, + DATABASE_DENY, + SCHEMA_ALLOW, + SCHEMA_DENY, + VIEW_ALLOW, + VIEW_DENY, + ], + }, + [BIGQUERY]: { + fields: [ + BIGQUERY_PROJECT_ID, + BIGQUERY_CREDENTIAL_PROJECT_ID, + BIGQUERY_PRIVATE_KEY, + BIGQUERY_PRIVATE_KEY_ID, + BIGQUERY_CLIENT_EMAIL, + BIGQUERY_CLIENT_ID, + ], + advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, UPSTREAM_LINEAGE_IN_REPORT], + filterFields: [TABLE_ALLOW, TABLE_DENY, SCHEMA_ALLOW, SCHEMA_DENY, VIEW_ALLOW, VIEW_DENY], + }, + [REDSHIFT]: { + fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD], + advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE], + filterFields: [TABLE_ALLOW, TABLE_DENY, SCHEMA_ALLOW, SCHEMA_DENY, VIEW_ALLOW, VIEW_DENY], + }, + [TABLEAU]: { + fields: [TABLEAU_CONNECTION_URI, TABLEAU_PROJECT, TABLEAU_SITE, TABLEAU_USERNAME, TABLEAU_PASSWORD], + filterFields: [], + advancedFields: [INGEST_TAGS, INGEST_OWNER], + }, + [LOOKER]: { + fields: [LOOKER_BASE_URL, LOOKER_CLIENT_ID, LOOKER_CLIENT_SECRET], + filterFields: [DASHBOARD_ALLOW, DASHBOARD_DENY, CHART_ALLOW, CHART_DENY], + advancedFields: [GITHUB_INFO_REPO, EXTRACT_USAGE_HISTORY, EXTRACT_OWNERS, SKIP_PERSONAL_FOLDERS], + }, +}; + +export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS)); diff --git a/datahub-web-react/src/app/ingest/source/builder/YamlEditor.tsx b/datahub-web-react/src/app/ingest/source/builder/YamlEditor.tsx index 1419d8e7f7dcf3..3550ef5e4fde3d 100644 --- a/datahub-web-react/src/app/ingest/source/builder/YamlEditor.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/YamlEditor.tsx @@ -1,5 +1,11 @@ import React from 'react'; -import Editor from '@monaco-editor/react'; +import Editor, { loader } from '@monaco-editor/react'; + +loader.config({ + paths: { + vs: `${process.env.PUBLIC_URL}/monaco-editor/vs`, + }, +}); type Props = { initialText: string; diff --git a/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx b/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx new file mode 100644 index 00000000000000..1cb38b6c87fbd8 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx @@ -0,0 +1,38 @@ +import { MockedProvider } from '@apollo/client/testing'; +import { render } from '@testing-library/react'; +import React from 'react'; +import { DefineRecipeStep } from '../DefineRecipeStep'; + +describe('DefineRecipeStep', () => { + it('should render the RecipeBuilder if the type is in CONNECTORS_WITH_FORM', () => { + const { getByText, queryByText } = render( + + {}} + goTo={() => {}} + submit={() => {}} + cancel={() => {}} + /> + , + ); + + expect(getByText('Connection')).toBeInTheDocument(); + expect(queryByText('For more information about how to configure a recipe')).toBeNull(); + }); + + it('should not render the RecipeBuilder if the type is not in CONNECTORS_WITH_FORM', () => { + const { getByText, queryByText } = render( + {}} + goTo={() => {}} + submit={() => {}} + cancel={() => {}} + />, + ); + + expect(getByText('Configure Postgres Recipe')).toBeInTheDocument(); + expect(queryByText('Connection')).toBeNull(); + }); +}); diff --git a/datahub-web-react/src/app/ingest/source/builder/types.ts b/datahub-web-react/src/app/ingest/source/builder/types.ts index 4936a13ae9ed8c..c2a2693c9eb4c9 100644 --- a/datahub-web-react/src/app/ingest/source/builder/types.ts +++ b/datahub-web-react/src/app/ingest/source/builder/types.ts @@ -21,7 +21,7 @@ export type StepProps = { updateState: (newState: SourceBuilderState) => void; goTo: (step: IngestionSourceBuilderStep) => void; prev?: () => void; - submit: () => void; + submit: (shouldRun?: boolean) => void; cancel: () => void; }; diff --git a/datahub-web-react/src/app/ingest/source/conf/azure/azure.ts b/datahub-web-react/src/app/ingest/source/conf/azure/azure.ts index 05c4b076c85741..1d30d9448d9f4c 100644 --- a/datahub-web-react/src/app/ingest/source/conf/azure/azure.ts +++ b/datahub-web-react/src/app/ingest/source/conf/azure/azure.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import azureLogo from '../../../../../images/azure-ad.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: azure-ad @@ -29,12 +27,7 @@ source: # users_pattern: # allow: # - ".*" -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const azureAdConfig: SourceConfig = { type: 'azure-ad', diff --git a/datahub-web-react/src/app/ingest/source/conf/bigquery/bigquery.ts b/datahub-web-react/src/app/ingest/source/conf/bigquery/bigquery.ts index 5140208a6a8209..e064b8a2ce2b16 100644 --- a/datahub-web-react/src/app/ingest/source/conf/bigquery/bigquery.ts +++ b/datahub-web-react/src/app/ingest/source/conf/bigquery/bigquery.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import bigqueryLogo from '../../../../../images/bigquerylogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: bigquery @@ -12,24 +10,23 @@ source: # Credentials credential: project_id: # Your BQ project id, e.g. sample_project_id - - # Add secret in Secrets Tab with the relevant names for each variable below - # Your BQ private key id, e.g. "d0121d0000882411234e11166c6aaa23ed5d74e0" private_key_id: "\${BQ_PRIVATE_KEY_ID}" - # Your BQ private key, e.g. "-----BEGIN PRIVATE KEY-----\\nMIIyourkey\\n-----END PRIVATE KEY-----\\n" private_key: "\${BQ_PRIVATE_KEY}" - client_email: # Your BQ client email, e.g. "test@suppproject-id-1234567.iam.gserviceaccount.com" client_id: # Your BQ client id, e.g. "123456678890" -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; + + include_table_lineage: true + include_view_lineage: true + profiling: + enabled: true + stateful_ingestion: + enabled: true +`; + +export const BIGQUERY = 'bigquery'; const bigqueryConfig: SourceConfig = { - type: 'bigquery', + type: BIGQUERY, placeholderRecipe, displayName: 'BigQuery', docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/bigquery/', diff --git a/datahub-web-react/src/app/ingest/source/conf/glue/glue.ts b/datahub-web-react/src/app/ingest/source/conf/glue/glue.ts index 8fb43812800299..449f76eea8afb8 100644 --- a/datahub-web-react/src/app/ingest/source/conf/glue/glue.ts +++ b/datahub-web-react/src/app/ingest/source/conf/glue/glue.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import glueLogo from '../../../../../images/gluelogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: glue @@ -24,12 +22,7 @@ source: # table_pattern: # allow: # - "avro" -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const glueConfig: SourceConfig = { type: 'glue', diff --git a/datahub-web-react/src/app/ingest/source/conf/hive/hive.ts b/datahub-web-react/src/app/ingest/source/conf/hive/hive.ts index 529e11919022c3..084a6676aa843c 100644 --- a/datahub-web-react/src/app/ingest/source/conf/hive/hive.ts +++ b/datahub-web-react/src/app/ingest/source/conf/hive/hive.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import hiveLogo from '../../../../../images/hivelogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: hive @@ -15,13 +13,7 @@ source: # Add secret in Secrets Tab with relevant names for each variable username: "\${HIVE_USERNAME}" # Your Hive username, e.g. admin password: "\${HIVE_PASSWORD}"# Your Hive password, e.g. password_01 - -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const hiveConfig: SourceConfig = { type: 'hive', diff --git a/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts b/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts index 1e2d0f47ff584e..e6bb0ee35b1646 100644 --- a/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts +++ b/datahub-web-react/src/app/ingest/source/conf/kafka/kafka.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import kafkaLogo from '../../../../../images/kafkalogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: kafka @@ -23,12 +21,7 @@ source: # Uncomment and add secrets in Secrets Tab # schema_registry_config: # basic.auth.user.info: "\${REGISTRY_API_KEY_ID}:\${REGISTRY_API_KEY_SECRET}" -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const kafkaConfig: SourceConfig = { type: 'kafka', diff --git a/datahub-web-react/src/app/ingest/source/conf/looker/looker.ts b/datahub-web-react/src/app/ingest/source/conf/looker/looker.ts index 3995e44e425039..c293f6c4af9931 100644 --- a/datahub-web-react/src/app/ingest/source/conf/looker/looker.ts +++ b/datahub-web-react/src/app/ingest/source/conf/looker/looker.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import lookerLogo from '../../../../../images/lookerlogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: looker @@ -14,15 +12,12 @@ source: # Add secret in Secrets Tab with relevant names for each variable client_id: "\${LOOKER_CLIENT_ID}" # Your Looker client id, e.g. admin client_secret: "\${LOOKER_CLIENT_SECRET}" # Your Looker password, e.g. password_01 -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; + +export const LOOKER = 'looker'; const lookerConfig: SourceConfig = { - type: 'looker', + type: LOOKER, placeholderRecipe, displayName: 'Looker', docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/looker/', diff --git a/datahub-web-react/src/app/ingest/source/conf/mongodb/mongodb.ts b/datahub-web-react/src/app/ingest/source/conf/mongodb/mongodb.ts index 043d1da23a7fc1..f4856adbede8e9 100644 --- a/datahub-web-react/src/app/ingest/source/conf/mongodb/mongodb.ts +++ b/datahub-web-react/src/app/ingest/source/conf/mongodb/mongodb.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import mongodbLogo from '../../../../../images/mongodblogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: mongodb @@ -19,12 +17,7 @@ source: enableSchemaInference: True useRandomSampling: True maxSchemaSize: 300 -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const mongoConfig: SourceConfig = { type: 'mongodb', diff --git a/datahub-web-react/src/app/ingest/source/conf/mysql/mysql.ts b/datahub-web-react/src/app/ingest/source/conf/mysql/mysql.ts index 82527baf189731..9f90677be6c19d 100644 --- a/datahub-web-react/src/app/ingest/source/conf/mysql/mysql.ts +++ b/datahub-web-react/src/app/ingest/source/conf/mysql/mysql.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import mysqlLogo from '../../../../../images/mysqllogo-2.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: mysql @@ -23,12 +21,7 @@ source: # Profiling profiling: enabled: false -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const mysqlConfig: SourceConfig = { type: 'mysql', diff --git a/datahub-web-react/src/app/ingest/source/conf/okta/okta.ts b/datahub-web-react/src/app/ingest/source/conf/okta/okta.ts index 7fb4144f9bfa1e..fc08db9312e0d1 100644 --- a/datahub-web-react/src/app/ingest/source/conf/okta/okta.ts +++ b/datahub-web-react/src/app/ingest/source/conf/okta/okta.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import oktaLogo from '../../../../../images/oktalogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: okta @@ -31,12 +29,7 @@ source: # Optional: Include deprovisioned or suspended Okta users in the ingestion. # include_deprovisioned_users = False # include_suspended_users = False -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const oktaConfig: SourceConfig = { type: 'okta', diff --git a/datahub-web-react/src/app/ingest/source/conf/oracle/oracle.ts b/datahub-web-react/src/app/ingest/source/conf/oracle/oracle.ts index 5cd4bdc9112578..df83dd1e70e087 100644 --- a/datahub-web-react/src/app/ingest/source/conf/oracle/oracle.ts +++ b/datahub-web-react/src/app/ingest/source/conf/oracle/oracle.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import oracleLogo from '../../../../../images/oraclelogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: oracle @@ -18,12 +16,7 @@ source: # Optional service name # service_name: # Your service name, e.g. svc # omit database if using this option -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const oracleConfig: SourceConfig = { type: 'oracle', diff --git a/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts b/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts index 6d5eb8898e96ca..93c4969988647e 100644 --- a/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts +++ b/datahub-web-react/src/app/ingest/source/conf/postgres/postgres.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import postgresLogo from '../../../../../images/postgreslogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: postgres @@ -23,12 +21,7 @@ source: # Profiling profiling: enabled: false -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; +`; const postgresConfig: SourceConfig = { type: 'postgres', diff --git a/datahub-web-react/src/app/ingest/source/conf/redshift/redshift.ts b/datahub-web-react/src/app/ingest/source/conf/redshift/redshift.ts index 93d37c9216d831..4a6858a3a1d48b 100644 --- a/datahub-web-react/src/app/ingest/source/conf/redshift/redshift.ts +++ b/datahub-web-react/src/app/ingest/source/conf/redshift/redshift.ts @@ -1,8 +1,6 @@ import { SourceConfig } from '../types'; import redshiftLogo from '../../../../../images/redshiftlogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: redshift @@ -16,22 +14,19 @@ source: username: "\${REDSHIFT_USERNAME}" # Your Redshift username, e.g. admin password: "\${REDSHIFT_PASSWORD}" # Your Redshift password, e.g. password_01 - # Options - include_tables: True - include_views: True - - # Profiling + table_lineage_mode: stl_scan_based + include_table_lineage: true + include_view_lineage: true profiling: - enabled: false -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms" - # Add a secret in secrets Tab - token: "\${GMS_TOKEN}"`; + enabled: true + stateful_ingestion: + enabled: true +`; + +export const REDSHIFT = 'redshift'; const redshiftConfig: SourceConfig = { - type: 'redshift', + type: REDSHIFT, placeholderRecipe, displayName: 'Redshift', docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/redshift/', diff --git a/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts b/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts index 4ac7fd7c450a6f..4edb6154cecb01 100644 --- a/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts +++ b/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts @@ -1,61 +1,25 @@ import { SourceConfig } from '../types'; import snowflakeLogo from '../../../../../images/snowflakelogo.png'; -const baseUrl = window.location.origin; - const placeholderRecipe = `\ source: type: snowflake config: - # Uncomment this section to provision the role required for ingestion - # provision_role: - # enabled: true - # dry_run: false - # run_ingestion: true - # admin_username: "\${SNOWFLAKE_ADMIN_USER}" - # admin_password: "\${SNOWFLAKE_ADMIN_PASS}" - - # Your Snowflake account name - # e.g. if URL is example48144.us-west-2.snowflakecomputing.com then use "example48144" - account_id: "example48144" - warehouse: # Your Snowflake warehouse name, e.g. "PROD_WH" - - # Credentials - username: "\${SNOWFLAKE_USER}" # Create a secret SNOWFLAKE_USER in secrets Tab - password: "\${SNOWFLAKE_PASS}" # Create a secret SNOWFLAKE_PASS in secrets Tab + account_id: "example_id" + warehouse: "example_warehouse" role: "datahub_role" + include_table_lineage: true + include_view_lineage: true + profiling: + enabled: true + stateful_ingestion: + enabled: true +`; - # Suggest to have this set to true initially to get all lineage - ignore_start_time_lineage: true - - # This is an alternative option to specify the start_time for lineage - # if you don't want to look back since beginning - # start_time: '2022-03-01T00:00:00Z' - - # Uncomment and change to only allow some database metadata to be ingested - # database_pattern: - # allow: - # - "^ACCOUNTING_DB$" - # - "^MARKETING_DB$" - - # Uncomment and change to deny some metadata from few schemas - # schema_pattern: - # deny: - # - "information_schema.*" - - # If you want to ingest only few tables with name revenue and sales - # table_pattern: - # allow: - # - ".*revenue" - # - ".*sales" - -sink: - type: datahub-rest - config: - server: "${baseUrl}/api/gms"`; +export const SNOWFLAKE = 'snowflake'; const snowflakeConfig: SourceConfig = { - type: 'snowflake', + type: SNOWFLAKE, placeholderRecipe, displayName: 'Snowflake', docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/snowflake/', diff --git a/datahub-web-react/src/app/ingest/source/conf/sources.tsx b/datahub-web-react/src/app/ingest/source/conf/sources.tsx index 0bc3538ebdfbde..9337ab59b86a77 100644 --- a/datahub-web-react/src/app/ingest/source/conf/sources.tsx +++ b/datahub-web-react/src/app/ingest/source/conf/sources.tsx @@ -15,6 +15,7 @@ import oktaConfig from './okta/okta'; import { SourceConfig } from './types'; import hiveConfig from './hive/hive'; import oracleConfig from './oracle/oracle'; +import tableauConfig from './tableau/tableau'; const baseUrl = window.location.origin; @@ -36,6 +37,7 @@ export const SOURCE_TEMPLATE_CONFIGS: Array = [ snowflakeConfig, kafkaConfig, lookerConfig, + tableauConfig, mysqlConfig, postgresConfig, mongoConfig, diff --git a/datahub-web-react/src/app/ingest/source/conf/tableau/tableau.ts b/datahub-web-react/src/app/ingest/source/conf/tableau/tableau.ts new file mode 100644 index 00000000000000..76ce171b2b92d0 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/conf/tableau/tableau.ts @@ -0,0 +1,35 @@ +import { SourceConfig } from '../types'; +import tableauLogo from '../../../../../images/tableaulogo.png'; + +const placeholderRecipe = `\ +source: + type: tableau + config: + # Coordinates + connect_uri: https://prod-ca-a.online.tableau.com + site: acryl + projects: ["default", "Project 2"] + + # Credentials + username: "\${TABLEAU_USER}" + password: "\${TABLEAU_PASSWORD}" + + # Options + ingest_tags: True + ingest_owner: True + default_schema_map: + mydatabase: public + anotherdatabase: anotherschema +`; + +export const TABLEAU = 'tableau'; + +const tableauConfig: SourceConfig = { + type: TABLEAU, + placeholderRecipe, + displayName: 'Tableau', + docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/tableau/', + logoUrl: tableauLogo, +}; + +export default tableauConfig; diff --git a/datahub-web-react/src/app/ingest/source/utils.ts b/datahub-web-react/src/app/ingest/source/utils.ts index 861b6d3e3d0387..7c50a0d2a21117 100644 --- a/datahub-web-react/src/app/ingest/source/utils.ts +++ b/datahub-web-react/src/app/ingest/source/utils.ts @@ -2,6 +2,8 @@ import YAML from 'yamljs'; import { CheckCircleOutlined, CloseCircleOutlined, LoadingOutlined } from '@ant-design/icons'; import { ANTD_GRAY, REDESIGN_COLORS } from '../../entity/shared/constants'; import { SOURCE_TEMPLATE_CONFIGS } from './conf/sources'; +import { EntityType, FacetMetadata } from '../../../types.generated'; +import { capitalizeFirstLetterOnly, pluralize } from '../../shared/textUtil'; export const sourceTypeToIconUrl = (type: string) => { return SOURCE_TEMPLATE_CONFIGS.find((config) => config.type === type)?.logoUrl; @@ -27,32 +29,88 @@ export const jsonToYaml = (json: string): string => { return yamlStr; }; +export const RUNNING = 'RUNNING'; +export const SUCCESS = 'SUCCESS'; +export const FAILURE = 'FAILURE'; +export const CANCELLED = 'CANCELLED'; + export const getExecutionRequestStatusIcon = (status: string) => { return ( - (status === 'RUNNING' && LoadingOutlined) || - (status === 'SUCCESS' && CheckCircleOutlined) || - (status === 'FAILURE' && CloseCircleOutlined) || - (status === 'CANCELLED' && CloseCircleOutlined) || + (status === RUNNING && LoadingOutlined) || + (status === SUCCESS && CheckCircleOutlined) || + (status === FAILURE && CloseCircleOutlined) || + (status === CANCELLED && CloseCircleOutlined) || undefined ); }; export const getExecutionRequestStatusDisplayText = (status: string) => { return ( - (status === 'RUNNING' && 'Running') || - (status === 'SUCCESS' && 'Succeeded') || - (status === 'FAILURE' && 'Failed') || - (status === 'CANCELLED' && 'Cancelled') || + (status === RUNNING && 'Running') || + (status === SUCCESS && 'Succeeded') || + (status === FAILURE && 'Failed') || + (status === CANCELLED && 'Cancelled') || status ); }; export const getExecutionRequestStatusDisplayColor = (status: string) => { return ( - (status === 'RUNNING' && REDESIGN_COLORS.BLUE) || - (status === 'SUCCESS' && 'green') || - (status === 'FAILURE' && 'red') || - (status === 'CANCELLED' && ANTD_GRAY[9]) || + (status === RUNNING && REDESIGN_COLORS.BLUE) || + (status === SUCCESS && 'green') || + (status === FAILURE && 'red') || + (status === CANCELLED && ANTD_GRAY[9]) || ANTD_GRAY[7] ); }; + +const ENTITIES_WITH_SUBTYPES = new Set([ + EntityType.Dataset.toLowerCase(), + EntityType.Container.toLowerCase(), + EntityType.Notebook.toLowerCase(), +]); + +type EntityTypeCount = { + count: number; + displayName: string; +}; + +/** + * Extract entity type counts to display in the ingestion summary. + * + * @param entityTypeFacets the filter facets for entity type. + * @param subTypeFacets the filter facets for sub types. + */ +export const extractEntityTypeCountsFromFacets = ( + entityTypeFacets: FacetMetadata, + subTypeFacets?: FacetMetadata | null, +): EntityTypeCount[] => { + const finalCounts: EntityTypeCount[] = []; + + if (subTypeFacets) { + subTypeFacets.aggregations.forEach((agg) => + finalCounts.push({ + count: agg.count, + displayName: pluralize(agg.count, capitalizeFirstLetterOnly(agg.value) || ''), + }), + ); + entityTypeFacets.aggregations + .filter((agg) => !ENTITIES_WITH_SUBTYPES.has(agg.value.toLowerCase())) + .forEach((agg) => + finalCounts.push({ + count: agg.count, + displayName: pluralize(agg.count, capitalizeFirstLetterOnly(agg.value) || ''), + }), + ); + } else { + // Only use Entity Types- no subtypes. + entityTypeFacets.aggregations.forEach((agg) => + finalCounts.push({ + count: agg.count, + displayName: pluralize(agg.count, capitalizeFirstLetterOnly(agg.value) || ''), + }), + ); + } + + return finalCounts; +}; diff --git a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx index c888d72bf5c047..19887f831081fc 100644 --- a/datahub-web-react/src/app/lineage/LineageEntityNode.tsx +++ b/datahub-web-react/src/app/lineage/LineageEntityNode.tsx @@ -10,7 +10,8 @@ import { ANTD_GRAY } from '../entity/shared/constants'; import { capitalizeFirstLetter } from '../shared/textUtil'; import { nodeHeightFromTitleLength } from './utils/nodeHeightFromTitleLength'; import { LineageExplorerContext } from './utils/LineageExplorerContext'; -import useLazyGetEntityQuery from './utils/useLazyGetEntityQuery'; +import { useGetEntityLineageLazyQuery } from '../../graphql/lineage.generated'; +import { useIsSeparateSiblingsMode } from '../entity/shared/siblingUtils'; const CLICK_DELAY_THRESHOLD = 1000; const DRAG_DISTANCE_THRESHOLD = 20; @@ -89,13 +90,18 @@ export default function LineageEntityNode({ const { expandTitles } = useContext(LineageExplorerContext); const [isExpanding, setIsExpanding] = useState(false); const [expandHover, setExpandHover] = useState(false); - const { getAsyncEntity, asyncData } = useLazyGetEntityQuery(); + const [getAsyncEntityLineage, { data: asyncLineageData }] = useGetEntityLineageLazyQuery(); + const isHideSiblingMode = useIsSeparateSiblingsMode(); useEffect(() => { - if (asyncData) { - onExpandClick(asyncData); + if (asyncLineageData && asyncLineageData.entity) { + const entityAndType = { + type: asyncLineageData.entity.type, + entity: { ...asyncLineageData.entity }, + } as EntityAndType; + onExpandClick(entityAndType); } - }, [asyncData, onExpandClick]); + }, [asyncLineageData, onExpandClick]); const entityRegistry = useEntityRegistry(); const unexploredHiddenChildren = @@ -112,6 +118,13 @@ export default function LineageEntityNode({ [], ); + let platformDisplayText = capitalizeFirstLetter(node.data.platform); + if (node.data.siblingPlatforms && !isHideSiblingMode) { + platformDisplayText = node.data.siblingPlatforms + .map((platform) => platform.properties?.displayName || platform.name) + .join(' & '); + } + const nodeHeight = nodeHeightFromTitleLength(expandTitles ? node.data.expandedName || node.data.name : undefined); return ( @@ -148,7 +161,10 @@ export default function LineageEntityNode({ onClick={() => { setIsExpanding(true); if (node.data.urn && node.data.type) { - getAsyncEntity(node.data.urn, node.data.type); + // getAsyncEntity(node.data.urn, node.data.type); + getAsyncEntityLineage({ + variables: { urn: node.data.urn, separateSiblings: isHideSiblingMode }, + }); } }} onMouseOver={() => { @@ -237,23 +253,42 @@ export default function LineageEntityNode({ // eslint-disable-next-line react/style-prop-object style={{ filter: isSelected ? 'url(#shadow1-selected)' : 'url(#shadow1)' }} /> - {node.data.icon ? ( + {node.data.siblingPlatforms && !isHideSiblingMode && ( + + + + + )} + {(!node.data.siblingPlatforms || isHideSiblingMode) && node.data.icon && ( - ) : ( - node.data.type && ( - - ) + )} + {!node.data.icon && (!node.data.siblingPlatforms || isHideSiblingMode) && node.data.type && ( + )} - {truncate(capitalizeFirstLetter(node.data.platform), 16)} + {truncate(platformDisplayText, 16)} {' '} |{' '} diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx index aedba2e1fe1561..284a3b1a5c7435 100644 --- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useHistory } from 'react-router'; import { Alert, Button, Drawer } from 'antd'; @@ -11,10 +11,11 @@ import CompactContext from '../shared/CompactContext'; import { EntityAndType, EntitySelectParams, FetchedEntities } from './types'; import LineageViz from './LineageViz'; import extendAsyncEntities from './utils/extendAsyncEntities'; -import useGetEntityQuery from './utils/useGetEntityQuery'; import { EntityType } from '../../types.generated'; import { capitalizeFirstLetter } from '../shared/textUtil'; import { ANTD_GRAY } from '../entity/shared/constants'; +import { GetEntityLineageQuery, useGetEntityLineageQuery } from '../../graphql/lineage.generated'; +import { useIsSeparateSiblingsMode } from '../entity/shared/siblingUtils'; const DEFAULT_DISTANCE_FROM_TOP = 106; @@ -45,6 +46,16 @@ function usePrevious(value) { return ref.current; } +export function getEntityAndType(lineageData?: GetEntityLineageQuery) { + if (lineageData && lineageData.entity) { + return { + type: lineageData.entity.type, + entity: { ...lineageData.entity }, + } as EntityAndType; + } + return null; +} + type Props = { urn: string; type: EntityType; @@ -55,13 +66,23 @@ export default function LineageExplorer({ urn, type }: Props) { const history = useHistory(); const entityRegistry = useEntityRegistry(); + const isHideSiblingMode = useIsSeparateSiblingsMode(); - const { loading, error, data } = useGetEntityQuery(urn, type); + const { loading, error, data } = useGetEntityLineageQuery({ + variables: { urn, separateSiblings: isHideSiblingMode }, + }); + + const entityData: EntityAndType | null | undefined = useMemo(() => getEntityAndType(data), [data]); const [isDrawerVisible, setIsDrawVisible] = useState(false); const [selectedEntity, setSelectedEntity] = useState(undefined); const [asyncEntities, setAsyncEntities] = useState({}); + // in the case that sibling mode changes, we want to clear out our cache of entities + useEffect(() => { + setAsyncEntities({}); + }, [isHideSiblingMode]); + const drawerRef: React.MutableRefObject = useRef(null); const maybeAddAsyncLoadedEntity = useCallback( @@ -89,10 +110,10 @@ export default function LineageExplorer({ urn, type }: Props) { }; useEffect(() => { - if (type && data) { - maybeAddAsyncLoadedEntity(data); + if (type && entityData && !loading) { + maybeAddAsyncLoadedEntity(entityData); } - }, [data, asyncEntities, setAsyncEntities, maybeAddAsyncLoadedEntity, urn, previousUrn, type]); + }, [entityData, setAsyncEntities, maybeAddAsyncLoadedEntity, urn, previousUrn, type, loading]); if (error || (!loading && !error && !data)) { return ; @@ -109,7 +130,7 @@ export default function LineageExplorer({ urn, type }: Props) { { setIsDrawVisible(true); setSelectedEntity(params); diff --git a/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx b/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx index ca8987b7e6497e..ffbc6bf39d1775 100644 --- a/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx +++ b/datahub-web-react/src/app/lineage/LineageVizInsideZoom.tsx @@ -1,8 +1,9 @@ import React, { SVGProps, useEffect, useMemo, useState } from 'react'; -import { PlusOutlined, MinusOutlined } from '@ant-design/icons'; +import { PlusOutlined, MinusOutlined, QuestionCircleOutlined } from '@ant-design/icons'; import styled from 'styled-components'; -import { Button, Switch } from 'antd'; +import { Button, Switch, Tooltip } from 'antd'; import { ProvidedZoom, TransformMatrix } from '@vx/zoom/lib/types'; +import { useHistory, useLocation } from 'react-router-dom'; import LineageTree from './LineageTree'; import constructTree from './utils/constructTree'; @@ -10,6 +11,8 @@ import { Direction, EntityAndType, EntitySelectParams, FetchedEntity } from './t import { useEntityRegistry } from '../useEntityRegistry'; import { ANTD_GRAY } from '../entity/shared/constants'; import { LineageExplorerContext } from './utils/LineageExplorerContext'; +import { useIsSeparateSiblingsMode } from '../entity/shared/siblingUtils'; +import { navigateToLineageUrl } from './utils/navigateToLineageUrl'; const ZoomContainer = styled.div` position: relative; @@ -32,11 +35,8 @@ const DisplayControls = styled.div` box-shadow: 0px 0px 4px 0px #0000001a; `; -const ControlsTitle = styled.div` - margin-bottom: 12px; -`; - const ControlsSwitch = styled(Switch)` + margin-top: 12px; margin-right: 8px; `; @@ -62,6 +62,10 @@ const RootSvg = styled.svg<{ isDragging: boolean } & SVGProps>` } `; +const ControlLabel = styled.span` + vertical-align: sub; +`; + type Props = { margin: { top: number; right: number; bottom: number; left: number }; entityAndType?: EntityAndType | null; @@ -78,6 +82,11 @@ type Props = { height: number; }; +const HelpIcon = styled(QuestionCircleOutlined)` + color: ${ANTD_GRAY[7]}; + padding-left: 4px; +`; + export default function LineageVizInsideZoom({ zoom, margin, @@ -91,10 +100,13 @@ export default function LineageVizInsideZoom({ height, }: Props) { const [draggedNodes, setDraggedNodes] = useState>({}); + const history = useHistory(); + const location = useLocation(); const [hoveredEntity, setHoveredEntity] = useState(undefined); const [isDraggingNode, setIsDraggingNode] = useState(false); const [showExpandedTitles, setShowExpandedTitles] = useState(false); + const isHideSiblingMode = useIsSeparateSiblingsMode(); const entityRegistry = useEntityRegistry(); @@ -131,12 +143,34 @@ export default function LineageVizInsideZoom({ - Controls - setShowExpandedTitles(checked)} - />{' '} - Show Full Titles +
Controls
+
+ setShowExpandedTitles(checked)} + />{' '} + Show Full Titles +
+
+ { + navigateToLineageUrl({ + location, + history, + isLineageMode: true, + isHideSiblingMode: !checked, + }); + }} + />{' '} + + Compress Lineage{' '} + + + + +
; upstreamChildren?: Array; + numUpstreamChildren?: number; downstreamChildren?: Array; + numDownstreamChildren?: number; fullyFetched?: boolean; platform?: string; status?: Maybe; + siblingPlatforms?: Maybe; }; export type NodeData = { @@ -55,6 +60,7 @@ export type NodeData = { countercurrentChildrenUrns?: string[]; platform?: string; status?: Maybe; + siblingPlatforms?: Maybe; }; export type VizNode = { @@ -129,3 +135,9 @@ export type EntityAndType = type: EntityType.MlprimaryKey; entity: MlPrimaryKey; }; + +export interface LineageResult { + urn: string; + upstream?: Maybe<{ __typename?: 'EntityLineageResult' } & FullLineageResultsFragment>; + downstream?: Maybe<{ __typename?: 'EntityLineageResult' } & FullLineageResultsFragment>; +} diff --git a/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts b/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts index 008b6ec78312f3..d1571c01529ebc 100644 --- a/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts +++ b/datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts @@ -61,6 +61,7 @@ export default function constructFetchedNode( children: [], platform: fetchedNode?.platform, status: fetchedNode.status, + siblingPlatforms: fetchedNode.siblingPlatforms, }; // eslint-disable-next-line no-param-reassign diff --git a/datahub-web-react/src/app/lineage/utils/constructTree.ts b/datahub-web-react/src/app/lineage/utils/constructTree.ts index b9f6891d6988bb..4207ef6a0f64ab 100644 --- a/datahub-web-react/src/app/lineage/utils/constructTree.ts +++ b/datahub-web-react/src/app/lineage/utils/constructTree.ts @@ -22,6 +22,7 @@ export default function constructTree( icon: fetchedEntity?.icon, platform: fetchedEntity?.platform, unexploredChildren: 0, + siblingPlatforms: fetchedEntity?.siblingPlatforms, }; const lineageConfig = entityRegistry.getLineageVizConfig(entityAndType.type, entityAndType.entity); let children: EntityAndType[] = []; diff --git a/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts b/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts index d5d3dc9904a0e9..92c2964b9b6f03 100644 --- a/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts +++ b/datahub-web-react/src/app/lineage/utils/navigateToLineageUrl.ts @@ -1,10 +1,12 @@ import * as QueryString from 'query-string'; import { RouteComponentProps } from 'react-router-dom'; +import { SEPARATE_SIBLINGS_URL_PARAM } from '../../entity/shared/siblingUtils'; export const navigateToLineageUrl = ({ location, history, isLineageMode, + isHideSiblingMode, }: { location: { search: string; @@ -12,12 +14,19 @@ export const navigateToLineageUrl = ({ }; history: RouteComponentProps['history']; isLineageMode: boolean; + isHideSiblingMode?: boolean; }) => { const parsedSearch = QueryString.parse(location.search, { arrayFormat: 'comma' }); - const newSearch = { + let newSearch: any = { ...parsedSearch, is_lineage_mode: isLineageMode, }; + if (isHideSiblingMode !== undefined) { + newSearch = { + ...newSearch, + [SEPARATE_SIBLINGS_URL_PARAM]: isHideSiblingMode, + }; + } const newSearchStringified = QueryString.stringify(newSearch, { arrayFormat: 'comma' }); history.push({ diff --git a/datahub-web-react/src/app/lineage/utils/useGetEntityQuery.ts b/datahub-web-react/src/app/lineage/utils/useGetEntityQuery.ts deleted file mode 100644 index 490f8fe6a337ea..00000000000000 --- a/datahub-web-react/src/app/lineage/utils/useGetEntityQuery.ts +++ /dev/null @@ -1,206 +0,0 @@ -import { useMemo } from 'react'; -import { useGetChartQuery } from '../../../graphql/chart.generated'; -import { useGetDashboardQuery } from '../../../graphql/dashboard.generated'; -import { useGetDatasetQuery } from '../../../graphql/dataset.generated'; -import { useGetDataJobQuery } from '../../../graphql/dataJob.generated'; -import { useGetMlFeatureTableQuery } from '../../../graphql/mlFeatureTable.generated'; -import { useGetMlFeatureQuery } from '../../../graphql/mlFeature.generated'; -import { useGetMlPrimaryKeyQuery } from '../../../graphql/mlPrimaryKey.generated'; -import { EntityType } from '../../../types.generated'; -import { EntityAndType } from '../types'; -import { useGetMlModelQuery } from '../../../graphql/mlModel.generated'; -import { useGetMlModelGroupQuery } from '../../../graphql/mlModelGroup.generated'; - -export default function useGetEntityQuery(urn: string, entityType?: EntityType) { - const allResults = { - [EntityType.Dataset]: useGetDatasetQuery({ - variables: { urn }, - skip: entityType !== EntityType.Dataset, - }), - [EntityType.Chart]: useGetChartQuery({ - variables: { urn }, - skip: entityType !== EntityType.Chart, - }), - [EntityType.Dashboard]: useGetDashboardQuery({ - variables: { urn }, - skip: entityType !== EntityType.Dashboard, - }), - [EntityType.DataJob]: useGetDataJobQuery({ - variables: { urn }, - skip: entityType !== EntityType.DataJob, - }), - [EntityType.MlfeatureTable]: useGetMlFeatureTableQuery({ - variables: { urn }, - skip: entityType !== EntityType.MlfeatureTable, - }), - [EntityType.Mlfeature]: useGetMlFeatureQuery({ - variables: { urn }, - skip: entityType !== EntityType.Mlfeature, - }), - [EntityType.MlprimaryKey]: useGetMlPrimaryKeyQuery({ - variables: { urn }, - skip: entityType !== EntityType.MlprimaryKey, - }), - [EntityType.Mlmodel]: useGetMlModelQuery({ - variables: { urn }, - skip: entityType !== EntityType.Mlmodel, - }), - [EntityType.MlmodelGroup]: useGetMlModelGroupQuery({ - variables: { urn }, - skip: entityType !== EntityType.MlmodelGroup, - }), - }; - - const returnEntityAndType: EntityAndType | undefined = useMemo(() => { - let returnData; - switch (entityType) { - case EntityType.Dataset: - returnData = allResults[EntityType.Dataset].data?.dataset; - if (returnData) { - return { - entity: returnData, - type: EntityType.Dataset, - } as EntityAndType; - } - break; - case EntityType.Chart: - returnData = allResults[EntityType.Chart]?.data?.chart; - if (returnData) { - return { - entity: returnData, - type: EntityType.Chart, - } as EntityAndType; - } - break; - case EntityType.Dashboard: - returnData = allResults[EntityType.Dashboard]?.data?.dashboard; - if (returnData) { - return { - entity: returnData, - type: EntityType.Dashboard, - } as EntityAndType; - } - break; - case EntityType.DataJob: - returnData = allResults[EntityType.DataJob]?.data?.dataJob; - if (returnData) { - return { - entity: returnData, - type: EntityType.DataJob, - } as EntityAndType; - } - break; - case EntityType.MlfeatureTable: - returnData = allResults[EntityType.MlfeatureTable]?.data?.mlFeatureTable; - if (returnData) { - return { - entity: returnData, - type: EntityType.MlfeatureTable, - } as EntityAndType; - } - break; - case EntityType.Mlfeature: - returnData = allResults[EntityType.Mlfeature]?.data?.mlFeature; - if (returnData) { - return { - entity: returnData, - type: EntityType.Mlfeature, - } as EntityAndType; - } - break; - case EntityType.MlprimaryKey: - returnData = allResults[EntityType.MlprimaryKey]?.data?.mlPrimaryKey; - if (returnData) { - return { - entity: returnData, - type: EntityType.MlprimaryKey, - } as EntityAndType; - } - break; - case EntityType.Mlmodel: - returnData = allResults[EntityType.Mlmodel]?.data?.mlModel; - if (returnData) { - return { - entity: returnData, - type: EntityType.Mlmodel, - } as EntityAndType; - } - break; - case EntityType.MlmodelGroup: - returnData = allResults[EntityType.MlmodelGroup]?.data?.mlModelGroup; - if (returnData) { - return { - entity: returnData, - type: EntityType.MlmodelGroup, - } as EntityAndType; - } - break; - default: - break; - } - return undefined; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - urn, - entityType, - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Dataset], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Chart], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Dashboard], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.DataJob], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.MlfeatureTable], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Mlmodel], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.MlmodelGroup], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Mlfeature], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.MlprimaryKey], - ]); - - const returnObject = useMemo(() => { - if (!entityType) { - return { - loading: false, - error: null, - data: null, - }; - } - - return { - data: returnEntityAndType, - loading: allResults[entityType].loading, - error: allResults[entityType].error, - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - urn, - entityType, - returnEntityAndType, - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Dataset], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Chart], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Dashboard], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.DataJob], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.MlfeatureTable], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Mlmodel], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.MlmodelGroup], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.Mlfeature], - // eslint-disable-next-line react-hooks/exhaustive-deps - allResults[EntityType.MlprimaryKey], - ]); - - return returnObject; -} diff --git a/datahub-web-react/src/app/lineage/utils/useLazyGetEntityQuery.ts b/datahub-web-react/src/app/lineage/utils/useLazyGetEntityQuery.ts deleted file mode 100644 index 558d0f7516b144..00000000000000 --- a/datahub-web-react/src/app/lineage/utils/useLazyGetEntityQuery.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { useCallback, useMemo, useState } from 'react'; -import { useGetChartLazyQuery } from '../../../graphql/chart.generated'; -import { useGetDashboardLazyQuery } from '../../../graphql/dashboard.generated'; -import { useGetDatasetLazyQuery } from '../../../graphql/dataset.generated'; -import { useGetDataJobLazyQuery } from '../../../graphql/dataJob.generated'; -import { useGetMlFeatureTableLazyQuery } from '../../../graphql/mlFeatureTable.generated'; -import { useGetMlFeatureLazyQuery } from '../../../graphql/mlFeature.generated'; -import { useGetMlPrimaryKeyLazyQuery } from '../../../graphql/mlPrimaryKey.generated'; -import { EntityType } from '../../../types.generated'; -import { EntityAndType } from '../types'; -import { useGetMlModelLazyQuery } from '../../../graphql/mlModel.generated'; -import { useGetMlModelGroupLazyQuery } from '../../../graphql/mlModelGroup.generated'; - -export default function useLazyGetEntityQuery() { - const [fetchedEntityType, setFetchedEntityType] = useState(undefined); - const [getAsyncDataset, { data: asyncDatasetData }] = useGetDatasetLazyQuery(); - const [getAsyncChart, { data: asyncChartData }] = useGetChartLazyQuery(); - const [getAsyncDashboard, { data: asyncDashboardData }] = useGetDashboardLazyQuery(); - const [getAsyncDataJob, { data: asyncDataJobData }] = useGetDataJobLazyQuery(); - const [getAsyncMLFeatureTable, { data: asyncMLFeatureTable }] = useGetMlFeatureTableLazyQuery(); - const [getAsyncMLFeature, { data: asyncMLFeature }] = useGetMlFeatureLazyQuery(); - const [getAsyncMLPrimaryKey, { data: asyncMLPrimaryKey }] = useGetMlPrimaryKeyLazyQuery(); - const [getAsyncMlModel, { data: asyncMlModel }] = useGetMlModelLazyQuery(); - const [getAsyncMlModelGroup, { data: asyncMlModelGroup }] = useGetMlModelGroupLazyQuery(); - - const getAsyncEntity = useCallback( - (urn: string, type: EntityType) => { - if (type === EntityType.Dataset) { - setFetchedEntityType(type); - getAsyncDataset({ variables: { urn } }); - } - if (type === EntityType.Chart) { - setFetchedEntityType(type); - getAsyncChart({ variables: { urn } }); - } - if (type === EntityType.Dashboard) { - setFetchedEntityType(type); - getAsyncDashboard({ variables: { urn } }); - } - if (type === EntityType.DataJob) { - setFetchedEntityType(type); - getAsyncDataJob({ variables: { urn } }); - } - if (type === EntityType.MlfeatureTable) { - setFetchedEntityType(type); - getAsyncMLFeatureTable({ variables: { urn } }); - } - if (type === EntityType.Mlfeature) { - setFetchedEntityType(type); - getAsyncMLFeature({ variables: { urn } }); - } - if (type === EntityType.MlprimaryKey) { - setFetchedEntityType(type); - getAsyncMLPrimaryKey({ variables: { urn } }); - } - if (type === EntityType.Mlmodel) { - setFetchedEntityType(type); - getAsyncMlModel({ variables: { urn } }); - } - if (type === EntityType.MlmodelGroup) { - setFetchedEntityType(type); - getAsyncMlModelGroup({ variables: { urn } }); - } - }, - [ - setFetchedEntityType, - getAsyncChart, - getAsyncDataset, - getAsyncDashboard, - getAsyncDataJob, - getAsyncMLFeatureTable, - getAsyncMLFeature, - getAsyncMLPrimaryKey, - getAsyncMlModel, - getAsyncMlModelGroup, - ], - ); - - const returnEntityAndType: EntityAndType | undefined = useMemo(() => { - let returnData; - switch (fetchedEntityType) { - case EntityType.Dataset: - returnData = asyncDatasetData?.dataset; - if (returnData) { - return { - entity: returnData, - type: EntityType.Dataset, - } as EntityAndType; - } - break; - case EntityType.Chart: - returnData = asyncChartData?.chart; - if (returnData) { - return { - entity: returnData, - type: EntityType.Chart, - } as EntityAndType; - } - break; - case EntityType.Dashboard: - returnData = asyncDashboardData?.dashboard; - if (returnData) { - return { - entity: returnData, - type: EntityType.Dashboard, - } as EntityAndType; - } - break; - case EntityType.DataJob: - returnData = asyncDataJobData?.dataJob; - if (returnData) { - return { - entity: returnData, - type: EntityType.DataJob, - } as EntityAndType; - } - break; - case EntityType.MlfeatureTable: - returnData = asyncMLFeatureTable?.mlFeatureTable; - if (returnData) { - return { - entity: returnData, - type: EntityType.MlfeatureTable, - } as EntityAndType; - } - break; - case EntityType.Mlfeature: - returnData = asyncMLFeature?.mlFeature; - if (returnData) { - return { - entity: returnData, - type: EntityType.Mlfeature, - } as EntityAndType; - } - break; - case EntityType.MlprimaryKey: - returnData = asyncMLPrimaryKey?.mlPrimaryKey; - if (returnData) { - return { - entity: returnData, - type: EntityType.MlprimaryKey, - } as EntityAndType; - } - break; - case EntityType.Mlmodel: - returnData = asyncMlModel?.mlModel; - if (returnData) { - return { - entity: returnData, - type: EntityType.Mlmodel, - } as EntityAndType; - } - break; - case EntityType.MlmodelGroup: - returnData = asyncMlModelGroup?.mlModelGroup; - if (returnData) { - return { - entity: returnData, - type: EntityType.MlmodelGroup, - } as EntityAndType; - } - break; - default: - break; - } - return undefined; - }, [ - asyncDatasetData, - asyncChartData, - asyncDashboardData, - asyncDataJobData, - fetchedEntityType, - asyncMLFeatureTable, - asyncMLFeature, - asyncMLPrimaryKey, - asyncMlModel, - asyncMlModelGroup, - ]); - - return { getAsyncEntity, asyncData: returnEntityAndType }; -} diff --git a/datahub-web-react/src/app/policy/ManagePolicies.tsx b/datahub-web-react/src/app/policy/ManagePolicies.tsx index f7522bce77172f..26a4eab51b7c84 100644 --- a/datahub-web-react/src/app/policy/ManagePolicies.tsx +++ b/datahub-web-react/src/app/policy/ManagePolicies.tsx @@ -248,12 +248,20 @@ export const ManagePolicies = () => { setFocusPolicyUrn(undefined); }; + const onEditPolicy = (policy: Policy) => { + setShowPolicyBuilderModal(true); + setFocusPolicyUrn(policy?.urn); + setFocusPolicy({ ...policy }); + }; + + // On Delete Policy handler const onRemovePolicy = (policy: Policy) => { Modal.confirm({ title: `Delete ${policy?.name}`, content: `Are you sure you want to remove policy?`, onOk() { deletePolicy({ variables: { urn: policy?.urn as string } }); // There must be a focus policy urn. + message.success('Successfully removed policy.'); setTimeout(function () { policiesRefetch(); }, 3000); @@ -266,16 +274,12 @@ export const ManagePolicies = () => { }); }; - const onEditPolicy = (policy: Policy) => { - setShowPolicyBuilderModal(true); - setFocusPolicyUrn(policy?.urn); - setFocusPolicy({ ...policy }); - }; - + // On Activate and deactivate Policy handler const onToggleActiveDuplicate = (policy: Policy) => { + const newState = policy?.state === PolicyState.Active ? PolicyState.Inactive : PolicyState.Active; const newPolicy = { ...policy, - state: policy?.state === PolicyState.Active ? PolicyState.Inactive : PolicyState.Active, + state: newState, }; updatePolicy({ variables: { @@ -283,12 +287,14 @@ export const ManagePolicies = () => { input: toPolicyInput(newPolicy), }, }); + message.success(`Successfully ${newState === PolicyState.Active ? 'activated' : 'deactivated'} policy.`); setTimeout(function () { policiesRefetch(); }, 3000); setShowViewPolicyModal(false); }; + // On Add/Update Policy handler const onSavePolicy = (savePolicy: Omit) => { if (focusPolicyUrn) { // If there's an URN associated with the focused policy, then we are editing an existing policy. @@ -351,6 +357,7 @@ export const ManagePolicies = () => { /> {record?.allUsers ? All Users : null} {record?.allGroups ? All Groups : null} + {record?.resourceOwners ? All Owners : null} ); }, @@ -407,6 +414,7 @@ export const ManagePolicies = () => { const tableData = policies?.map((policy) => ({ allGroups: policy?.actors?.allGroups, allUsers: policy?.actors?.allUsers, + resourceOwners: policy?.actors?.resourceOwners, description: policy?.description, editable: policy?.editable, name: policy?.name, diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index 91a0b30c987995..d0c07082a0e345 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -1,7 +1,8 @@ import React, { ReactNode } from 'react'; -import { Tooltip, Typography } from 'antd'; +import { Button, Divider, Tooltip, Typography } from 'antd'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; +import { ArrowRightOutlined } from '@ant-design/icons'; import { GlobalTags, @@ -9,12 +10,12 @@ import { GlossaryTerms, SearchInsight, Container, - Domain, ParentContainersResult, + Maybe, + CorpUser, + Deprecation, + Domain, } from '../../types.generated'; -import { useEntityRegistry } from '../useEntityRegistry'; - -import AvatarsGroup from '../shared/avatar/AvatarsGroup'; import TagTermGroup from '../shared/tags/TagTermGroup'; import { ANTD_GRAY } from '../entity/shared/constants'; import NoMarkdownViewer from '../entity/shared/components/styled/StripMarkdownText'; @@ -23,6 +24,8 @@ import { useEntityData } from '../entity/shared/EntityContext'; import PlatformContentView from '../entity/shared/containers/profile/header/PlatformContent/PlatformContentView'; import { useParentContainersTruncation } from '../entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer'; import EntityCount from '../entity/shared/containers/profile/header/EntityCount'; +import { ExpandedActorGroup } from '../entity/shared/components/styled/ExpandedActorGroup'; +import { DeprecationPill } from '../entity/shared/components/styled/DeprecationPill'; const PreviewContainer = styled.div` display: flex; @@ -31,8 +34,13 @@ const PreviewContainer = styled.div` align-items: center; `; -const PreviewWrapper = styled.div` - width: 100%; +const LeftColumn = styled.div` + max-width: 60%; +`; + +const RightColumn = styled.div` + max-width: 40%; + display: flex; `; const TitleContainer = styled.div` @@ -44,8 +52,16 @@ const TitleContainer = styled.div` } `; +const EntityTitleContainer = styled.div` + display: flex; + align-items: center; +`; + const EntityTitle = styled(Typography.Text)<{ $titleSizePx?: number }>` display: block; + &&&:hover { + text-decoration: underline; + } &&& { margin-right 8px; @@ -76,14 +92,11 @@ const DescriptionContainer = styled.div` margin-bottom: 8px; `; -const AvatarContainer = styled.div` - margin-right: 32px; -`; - const TagContainer = styled.div` display: inline-flex; margin-left: 0px; margin-top: 3px; + flex-wrap: wrap; `; const TagSeparator = styled.div` @@ -107,6 +120,39 @@ const InsightIconContainer = styled.span` margin-right: 4px; `; +const ExternalUrlContainer = styled.span` + font-size: 12px; +`; + +const ExternalUrlButton = styled(Button)` + > :hover { + text-decoration: underline; + } + &&& { + padding-bottom: 0px; + } + padding-left: 12px; + padding-right: 12px; +`; + +const UserListContainer = styled.div` + display: flex; + flex-direction: column; + justify-content: right; + margin-right: 8px; +`; + +const UserListDivider = styled(Divider)` + padding: 4px; + height: auto; +`; + +const UserListTitle = styled(Typography.Text)` + text-align: right; + margin-bottom: 10px; + padding-right: 12px; +`; + interface Props { name: string; logoUrl?: string; @@ -117,14 +163,20 @@ interface Props { typeIcon?: JSX.Element; platform?: string; platformInstanceId?: string; + platforms?: Maybe[]; + logoUrls?: Maybe[]; qualifier?: string | null; tags?: GlobalTags; owners?: Array | null; + deprecation?: Deprecation | null; + topUsers?: Array | null; + externalUrl?: string | null; + subHeader?: React.ReactNode; snippet?: React.ReactNode; insights?: Array | null; glossaryTerms?: GlossaryTerms; container?: Container; - domain?: Domain | null; + domain?: Domain | undefined | null; entityCount?: number; dataTestID?: string; titleSizePx?: number; @@ -150,22 +202,27 @@ export default function DefaultPreviewCard({ qualifier, tags, owners, + topUsers, + subHeader, snippet, insights, glossaryTerms, domain, container, + deprecation, entityCount, titleSizePx, dataTestID, + externalUrl, onClick, degree, parentContainers, + platforms, + logoUrls, }: Props) { // sometimes these lists will be rendered inside an entity container (for example, in the case of impact analysis) // in those cases, we may want to enrich the preview w/ context about the container entity const { entityData } = useEntityData(); - const entityRegistry = useEntityRegistry(); const insightViews: Array = [ ...(insights?.map((insight) => ( <> @@ -182,37 +239,55 @@ export default function DefaultPreviewCard({ const { parentContainersRef, areContainersTruncated } = useParentContainersTruncation(container); + const onPreventMouseDown = (event) => { + event.preventDefault(); + event.stopPropagation(); + }; + return ( - - + + - - - - {name || ' '} - - {degree !== undefined && degree !== null && ( - - {getNumberWithOrdinal(degree)} - + + + + + {name || ' '} + + + {deprecation?.deprecated && } + {externalUrl && ( + + + View in {platform} + + )} - {!!degree && entityCount && } - - + + + {degree !== undefined && degree !== null && ( + + {getNumberWithOrdinal(degree)} + + )} + {!!degree && entityCount && } + {description && description.length > 0 && ( @@ -228,11 +303,7 @@ export default function DefaultPreviewCard({ {hasTags && } )} - {owners && owners.length > 0 && ( - - - - )} + {subHeader} {insightViews.length > 0 && ( {insightViews.map((insightView, index) => ( @@ -243,7 +314,28 @@ export default function DefaultPreviewCard({ ))} )} - + + + {topUsers && topUsers?.length > 0 && ( + <> + + Top Users +
+ +
+ + + )} + {(topUsers?.length || 0) > 0 && (owners?.length || 0) > 0 && } + {owners && owners?.length > 0 && ( + + Owners +
+ owner.owner)} max={2} /> +
+ + )} + ); } diff --git a/datahub-web-react/src/app/recommendations/renderer/component/CompactEntityNameList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/CompactEntityNameList.tsx index 176a9b4126d919..c3a116f6f4960a 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/CompactEntityNameList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/CompactEntityNameList.tsx @@ -1,5 +1,7 @@ +import { Tooltip } from 'antd'; import React from 'react'; -import { Entity } from '../../../../types.generated'; +import { useHistory } from 'react-router'; +import { Entity, SearchResult } from '../../../../types.generated'; import { IconStyleType } from '../../../entity/Entity'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { EntityPreviewTag } from './EntityPreviewTag'; @@ -7,9 +9,13 @@ import { EntityPreviewTag } from './EntityPreviewTag'; type Props = { entities: Array; onClick?: (index: number) => void; + linkUrlParams?: Record; + showTooltips?: boolean; }; -export const CompactEntityNameList = ({ entities, onClick }: Props) => { +export const CompactEntityNameList = ({ entities, onClick, linkUrlParams, showTooltips = true }: Props) => { const entityRegistry = useEntityRegistry(); + const history = useHistory(); + return ( <> {entities.map((entity, index) => { @@ -17,15 +23,44 @@ export const CompactEntityNameList = ({ entities, onClick }: Props) => { const platformLogoUrl = genericProps?.platform?.properties?.logoUrl; const displayName = entityRegistry.getDisplayName(entity.type, entity); const fallbackIcon = entityRegistry.getIcon(entity.type, 12, IconStyleType.ACCENT); - const url = entityRegistry.getEntityUrl(entity.type, entity.urn); + const url = entityRegistry.getEntityUrl(entity.type, entity.urn, linkUrlParams); return ( - onClick?.(index)} - /> + { + // prevents the search links from taking over + e.preventDefault(); + history.push(url); + }} + > + + {entityRegistry.renderSearchResult(entity.type, { + entity, + matchedFields: [], + } as SearchResult)} + + } + > + + platform.properties?.logoUrl, + )} + logoComponent={fallbackIcon} + onClick={() => onClick?.(index)} + /> + + + ); })} diff --git a/datahub-web-react/src/app/recommendations/renderer/component/EntityNameList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/EntityNameList.tsx index bbbcd26da82b5c..93956fbf214e74 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/EntityNameList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/EntityNameList.tsx @@ -1,13 +1,20 @@ import React from 'react'; -import { Divider, List } from 'antd'; +import { Divider, List, Checkbox } from 'antd'; import styled from 'styled-components'; import { Entity } from '../../../../types.generated'; import { useEntityRegistry } from '../../../useEntityRegistry'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { IconStyleType } from '../../../entity/Entity'; import { capitalizeFirstLetter } from '../../../shared/textUtil'; +import { EntityAndType } from '../../../entity/shared/types'; + +const StyledCheckbox = styled(Checkbox)` + margin-right: 12px; +`; const StyledList = styled(List)` + overflow-y: scroll; + height: 100%; margin-top: -1px; box-shadow: ${(props) => props.theme.styles['box-shadow']}; flex: 1; @@ -37,11 +44,13 @@ const StyledList = styled(List)` } ` as typeof List; -const ListItem = styled.div` +const ListItem = styled.div<{ isSelectMode: boolean }>` padding-right: 40px; - padding-left: 40px; + padding-left: ${(props) => (props.isSelectMode ? '20px' : '40px')}; padding-top: 16px; padding-bottom: 8px; + display: flex; + align-items: center; `; const ThinDivider = styled(Divider)` @@ -60,10 +69,24 @@ type Props = { additionalPropertiesList?: Array; entities: Array; onClick?: (index: number) => void; + isSelectMode?: boolean; + selectedEntities?: EntityAndType[]; + setSelectedEntities?: (entities: EntityAndType[]) => any; + bordered?: boolean; }; -export const EntityNameList = ({ additionalPropertiesList, entities, onClick }: Props) => { +export const EntityNameList = ({ + additionalPropertiesList, + entities, + onClick, + isSelectMode, + selectedEntities = [], + setSelectedEntities, + bordered = true, +}: Props) => { const entityRegistry = useEntityRegistry(); + const selectedEntityUrns = selectedEntities?.map((entity) => entity.urn) || []; + if ( additionalPropertiesList?.length !== undefined && additionalPropertiesList.length > 0 && @@ -74,9 +97,21 @@ export const EntityNameList = ({ additionalPropertiesList, entities, onClick }: { additionalPropertiesList, entities }, ); } + + /** + * Invoked when a new entity is selected. Simply updates the state of the list of selected entities. + */ + const onSelectEntity = (selectedEntity: EntityAndType, selected: boolean) => { + if (selected) { + setSelectedEntities?.([...selectedEntities, selectedEntity]); + } else { + setSelectedEntities?.(selectedEntities?.filter((entity) => entity.urn !== selectedEntity.urn) || []); + } + }; + return ( { const additionalProperties = additionalPropertiesList?.[index]; @@ -91,9 +126,18 @@ export const EntityNameList = ({ additionalPropertiesList, entities, onClick }: const fallbackIcon = entityRegistry.getIcon(entity.type, 18, IconStyleType.ACCENT); const subType = genericProps?.subTypes?.typeNames?.length && genericProps?.subTypes?.typeNames[0]; const entityCount = genericProps?.entityCount; + const deprecation = genericProps?.deprecation; return ( <> - + + {isSelectMode && ( + = 0} + onChange={(e) => + onSelectEntity({ urn: entity.urn, type: entity.type }, e.target.checked) + } + /> + )} onClick?.(index)} entityCount={entityCount} degree={additionalProperties?.degree} + deprecation={deprecation} /> diff --git a/datahub-web-react/src/app/recommendations/renderer/component/EntityPreviewTag.tsx b/datahub-web-react/src/app/recommendations/renderer/component/EntityPreviewTag.tsx index f760282ef4ed84..7f104c0e0cad77 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/EntityPreviewTag.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/EntityPreviewTag.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { Image, Tag } from 'antd'; import styled from 'styled-components'; import { Link } from 'react-router-dom'; +import { Maybe } from 'graphql/jsutils/Maybe'; const EntityTag = styled(Tag)` margin: 4px; @@ -37,20 +38,38 @@ type Props = { displayName: string; url: string; platformLogoUrl?: string; + platformLogoUrls?: Maybe[]; logoComponent?: React.ReactNode; onClick?: () => void; }; -export const EntityPreviewTag = ({ displayName, url, platformLogoUrl, logoComponent, onClick }: Props) => { +export const EntityPreviewTag = ({ + displayName, + url, + platformLogoUrl, + platformLogoUrls, + logoComponent, + onClick, +}: Props) => { return ( - {(platformLogoUrl && ) || + {(!!platformLogoUrl && !platformLogoUrls && ( + + )) || + (!!platformLogoUrls && + platformLogoUrls.slice(0, 2).map((platformLogoUrlsEntry) => ( + <> + + + ))) || logoComponent} - {displayName} + + {displayName} + diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx index fda6d7db772e5c..1d9162a84344ba 100644 --- a/datahub-web-react/src/app/search/SearchBar.tsx +++ b/datahub-web-react/src/app/search/SearchBar.tsx @@ -308,7 +308,7 @@ export const SearchBar = ({ ); } else { // Navigate directly to the entity profile. - history.push(getEntityPath(option.type, value, entityRegistry, false)); + history.push(getEntityPath(option.type, value, entityRegistry, false, false)); } }} onSearch={(value: string) => onQueryChange(value)} diff --git a/datahub-web-react/src/app/search/SearchFilters.tsx b/datahub-web-react/src/app/search/SearchFilters.tsx index 5cec5336b2933d..309533dab4c363 100644 --- a/datahub-web-react/src/app/search/SearchFilters.tsx +++ b/datahub-web-react/src/app/search/SearchFilters.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from 'react'; import { FacetMetadata } from '../../types.generated'; import { SearchFilter } from './SearchFilter'; -const TOP_FILTERS = ['entity', 'tags', 'glossaryTerms', 'domains', 'owners']; +const TOP_FILTERS = ['degree', 'entity', 'tags', 'glossaryTerms', 'domains', 'owners']; export const SearchFilterWrapper = styled.div` max-height: 100%; diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx index c6b8e67e407b32..d1d96c2c08177f 100644 --- a/datahub-web-react/src/app/search/SearchHeader.tsx +++ b/datahub-web-react/src/app/search/SearchHeader.tsx @@ -10,7 +10,7 @@ import EntityRegistry from '../entity/EntityRegistry'; import AdhocLink from '../create/AdhocLink'; import HelpLink from '../shared/admin/HelpLink'; import { ANTD_GRAY } from '../entity/shared/constants'; -import { AdminHeaderLinks } from '../shared/admin/AdminHeaderLinks'; +import { HeaderLinks } from '../shared/admin/HeaderLinks'; import ContactLink from '../shared/admin/ContactLink'; import { useAppConfig } from '../useAppConfig'; import { DEFAULT_APP_CONFIG } from '../../appConfigContext'; @@ -110,7 +110,7 @@ export const SearchHeader = ({ - +
diff --git a/datahub-web-react/src/app/search/SearchPage.tsx b/datahub-web-react/src/app/search/SearchPage.tsx index 589b36b7444d6e..ccc3f2240eeb53 100644 --- a/datahub-web-react/src/app/search/SearchPage.tsx +++ b/datahub-web-react/src/app/search/SearchPage.tsx @@ -1,9 +1,7 @@ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import * as QueryString from 'query-string'; import { useHistory, useLocation, useParams } from 'react-router'; import { Alert } from 'antd'; - -import { SearchablePage } from './SearchablePage'; import { useEntityRegistry } from '../useEntityRegistry'; import { FacetFilterInput, EntityType } from '../../types.generated'; import useFilters from './utils/useFilters'; @@ -14,6 +12,7 @@ import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.genera import { SearchCfg } from '../../conf'; import { ENTITY_FILTER_NAME } from './utils/constants'; import { GetSearchResultsParams } from '../entity/shared/components/styled/search/types'; +import { EntityAndType } from '../entity/shared/types'; type SearchPageParams = { type?: string; @@ -39,18 +38,34 @@ export const SearchPage = () => { .filter((filter) => filter.field === ENTITY_FILTER_NAME) .map((filter) => filter.value.toUpperCase() as EntityType); - const { data, loading, error } = useGetSearchResultsForMultipleQuery({ + const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE); + const [isSelectMode, setIsSelectMode] = useState(false); + const [selectedEntities, setSelectedEntities] = useState([]); + + const { + data, + loading, + error, + refetch: realRefetch, + } = useGetSearchResultsForMultipleQuery({ variables: { input: { types: entityFilters, query, - start: (page - 1) * SearchCfg.RESULTS_PER_PAGE, - count: SearchCfg.RESULTS_PER_PAGE, + start: (page - 1) * numResultsPerPage, + count: numResultsPerPage, filters: filtersWithoutEntities, }, }, }); + const searchResultEntities = + data?.searchAcrossEntities?.searchResults?.map((result) => ({ + urn: result.entity.urn, + type: result.entity.type, + })) || []; + const searchResultUrns = searchResultEntities.map((entity) => entity.urn); + // we need to extract refetch on its own so paging thru results for csv download // doesnt also update search results const { refetch } = useGetSearchResultsForMultipleQuery({ @@ -69,6 +84,36 @@ export const SearchPage = () => { return refetch(variables).then((res) => res.data.searchAcrossEntities); }; + const onChangeFilters = (newFilters: Array) => { + navigateToSearchUrl({ type: activeType, query, page: 1, filters: newFilters, history }); + }; + + const onChangePage = (newPage: number) => { + navigateToSearchUrl({ type: activeType, query, page: newPage, filters, history }); + }; + + /** + * Invoked when the "select all" checkbox is clicked. + * + * This method either adds the entire current page of search results to + * the list of selected entities, or removes the current page from the set of selected entities. + */ + const onChangeSelectAll = (selected: boolean) => { + if (selected) { + // Add current page of urns to the master selected entity list + const entitiesToAdd = searchResultEntities.filter( + (entity) => + selectedEntities.findIndex( + (element) => element.urn === entity.urn && element.type === entity.type, + ) < 0, + ); + setSelectedEntities(Array.from(new Set(selectedEntities.concat(entitiesToAdd)))); + } else { + // Filter out the current page of entity urns from the list + setSelectedEntities(selectedEntities.filter((entity) => searchResultUrns.indexOf(entity.urn) === -1)); + } + }; + useEffect(() => { if (!loading) { analytics.event({ @@ -79,30 +124,19 @@ export const SearchPage = () => { } }, [query, data, loading]); - const onSearch = (q: string, type?: EntityType) => { - if (q.trim().length === 0) { - return; - } - analytics.event({ - type: EventType.SearchEvent, - query: q, - entityTypeFilter: activeType, - pageNumber: 1, - originPath: window.location.pathname, - }); - navigateToSearchUrl({ type: type || activeType, query: q, page: 1, history }); - }; - - const onChangeFilters = (newFilters: Array) => { - navigateToSearchUrl({ type: activeType, query, page: 1, filters: newFilters, history }); - }; + useEffect(() => { + // When the query changes, then clear the select mode state + setIsSelectMode(false); + }, [query]); - const onChangePage = (newPage: number) => { - navigateToSearchUrl({ type: activeType, query, page: newPage, filters, history }); - }; + useEffect(() => { + if (!isSelectMode) { + setSelectedEntities([]); + } + }, [isSelectMode]); return ( - + <> {!loading && error && ( )} @@ -118,7 +152,15 @@ export const SearchPage = () => { loading={loading} onChangeFilters={onChangeFilters} onChangePage={onChangePage} + numResultsPerPage={numResultsPerPage} + setNumResultsPerPage={setNumResultsPerPage} + isSelectMode={isSelectMode} + selectedEntities={selectedEntities} + setSelectedEntities={setSelectedEntities} + setIsSelectMode={setIsSelectMode} + onChangeSelectAll={onChangeSelectAll} + refetch={realRefetch} /> - + ); }; diff --git a/datahub-web-react/src/app/search/SearchResultList.tsx b/datahub-web-react/src/app/search/SearchResultList.tsx new file mode 100644 index 00000000000000..4197ce190b963b --- /dev/null +++ b/datahub-web-react/src/app/search/SearchResultList.tsx @@ -0,0 +1,147 @@ +import React from 'react'; +import { Button, Checkbox, Divider, Empty, List, ListProps } from 'antd'; +import styled from 'styled-components'; +import { useHistory } from 'react-router'; +import { RocketOutlined } from '@ant-design/icons'; +import { navigateToSearchUrl } from './utils/navigateToSearchUrl'; +import { ANTD_GRAY } from '../entity/shared/constants'; +import { CombinedSearchResult, SEPARATE_SIBLINGS_URL_PARAM } from '../entity/shared/siblingUtils'; +import { CompactEntityNameList } from '../recommendations/renderer/component/CompactEntityNameList'; +import { useEntityRegistry } from '../useEntityRegistry'; +import { SearchResult } from '../../types.generated'; +import analytics, { EventType } from '../analytics'; +import { EntityAndType } from '../entity/shared/types'; + +const ResultList = styled(List)` + &&& { + width: 100%; + border-color: ${(props) => props.theme.styles['border-color-base']}; + margin-top: 8px; + padding: 16px 32px; + border-radius: 0px; + } +`; + +const StyledCheckbox = styled(Checkbox)` + margin-right: 12px; +`; + +const NoDataContainer = styled.div` + > div { + margin-top: 28px; + margin-bottom: 28px; + } +`; + +const ThinDivider = styled(Divider)` + margin-top: 16px; + margin-bottom: 16px; +`; + +const SiblingResultContainer = styled.div` + margin-top: 6px; +`; + +const ListItem = styled.div<{ isSelectMode: boolean }>` + display: flex; + align-items: center; + padding: 0px; +`; + +type Props = { + query: string; + searchResults: CombinedSearchResult[]; + totalResultCount: number; + isSelectMode: boolean; + selectedEntities: EntityAndType[]; + setSelectedEntities: (entities: EntityAndType[]) => any; +}; + +export const SearchResultList = ({ + query, + searchResults, + totalResultCount, + isSelectMode, + selectedEntities, + setSelectedEntities, +}: Props) => { + const history = useHistory(); + const entityRegistry = useEntityRegistry(); + const selectedEntityUrns = selectedEntities.map((entity) => entity.urn); + + const onClickResult = (result: SearchResult, index: number) => { + analytics.event({ + type: EventType.SearchResultClickEvent, + query, + entityUrn: result.entity.urn, + entityType: result.entity.type, + index, + total: totalResultCount, + }); + }; + + /** + * Invoked when a new entity is selected. Simply updates the state of the list of selected entities. + */ + const onSelectEntity = (selectedEntity: EntityAndType, selected: boolean) => { + if (selected) { + setSelectedEntities?.([...selectedEntities, selectedEntity]); + } else { + setSelectedEntities?.(selectedEntities?.filter((entity) => entity.urn !== selectedEntity.urn) || []); + } + }; + + return ( + <> + >> + dataSource={searchResults} + split={false} + locale={{ + emptyText: ( + + + + + ), + }} + renderItem={(item, index) => ( + <> + onClickResult(item, index)} + // class name for counting in test purposes only + className="test-search-result" + > + {isSelectMode && ( + = 0} + onChange={(e) => + onSelectEntity( + { urn: item.entity.urn, type: item.entity.type }, + e.target.checked, + ) + } + /> + )} + {entityRegistry.renderSearchResult(item.entity.type, item)} + + {item.matchedEntities && item.matchedEntities.length > 0 && ( + + + + )} + + + )} + /> + + ); +}; diff --git a/datahub-web-react/src/app/search/SearchResults.tsx b/datahub-web-react/src/app/search/SearchResults.tsx index 8c651ba8b08dcc..5e426c83c990a4 100644 --- a/datahub-web-react/src/app/search/SearchResults.tsx +++ b/datahub-web-react/src/app/search/SearchResults.tsx @@ -1,8 +1,6 @@ import React from 'react'; -import { Button, Divider, Empty, List, ListProps, Pagination, Typography } from 'antd'; +import { Pagination, Typography } from 'antd'; import styled from 'styled-components'; -import { RocketOutlined } from '@ant-design/icons'; -import { useHistory } from 'react-router'; import { Message } from '../shared/Message'; import { Entity, @@ -11,29 +9,19 @@ import { FacetMetadata, MatchedField, SearchAcrossEntitiesInput, - SearchResult, } from '../../types.generated'; import { SearchFilters } from './SearchFilters'; -import { useEntityRegistry } from '../useEntityRegistry'; -import analytics from '../analytics/analytics'; -import { EventType } from '../analytics'; import { SearchCfg } from '../../conf'; -import { navigateToSearchUrl } from './utils/navigateToSearchUrl'; -import { ANTD_GRAY } from '../entity/shared/constants'; import { SearchResultsRecommendations } from './SearchResultsRecommendations'; import { useGetAuthenticatedUser } from '../useGetAuthenticatedUser'; import { SearchResultsInterface } from '../entity/shared/components/styled/search/types'; import SearchExtendedMenu from '../entity/shared/components/styled/search/SearchExtendedMenu'; - -const ResultList = styled(List)` - &&& { - width: 100%; - border-color: ${(props) => props.theme.styles['border-color-base']}; - margin-top: 8px; - padding: 16px 32px; - border-radius: 0px; - } -`; +import { combineSiblingsInSearchResults } from '../entity/shared/siblingUtils'; +import { SearchSelectBar } from '../entity/shared/components/styled/search/SearchSelectBar'; +import { SearchResultList } from './SearchResultList'; +import { isListSubset } from '../entity/shared/utils'; +import TabToolbar from '../entity/shared/components/styled/TabToolbar'; +import { EntityAndType } from '../entity/shared/types'; const SearchBody = styled.div` display: flex; @@ -62,12 +50,14 @@ const PaginationControlContainer = styled.div` `; const PaginationInfoContainer = styled.div` - margin-top: 15px; - padding-left: 16px; + padding-left: 32px; + padding-right: 32px; + height: 47px; border-bottom: 1px solid; border-color: ${(props) => props.theme.styles['border-color-base']}; display: flex; justify-content: space-between; + align-items: center; `; const FiltersHeader = styled.div` @@ -89,26 +79,17 @@ const SearchFilterContainer = styled.div` padding-top: 10px; `; -const NoDataContainer = styled.div` - > div { - margin-top: 28px; - margin-bottom: 28px; - } -`; - -const ThinDivider = styled(Divider)` - margin-top: 16px; - margin-bottom: 16px; -`; - const SearchResultsRecommendationsContainer = styled.div` margin-top: 40px; `; -const SearchMenuContainer = styled.div` - margin-right: 10px; +const StyledTabToolbar = styled(TabToolbar)` + padding-left: 32px; + padding-right: 32px; `; +const SearchMenuContainer = styled.div``; + interface Props { query: string; page: number; @@ -131,6 +112,14 @@ interface Props { }) => Promise; entityFilters: EntityType[]; filtersWithoutEntities: FacetFilterInput[]; + numResultsPerPage: number; + setNumResultsPerPage: (numResults: number) => void; + isSelectMode: boolean; + selectedEntities: EntityAndType[]; + setSelectedEntities: (entities: EntityAndType[]) => void; + setIsSelectMode: (showSelectMode: boolean) => any; + onChangeSelectAll: (selected: boolean) => void; + refetch: () => void; } export const SearchResults = ({ @@ -145,31 +134,24 @@ export const SearchResults = ({ callSearchOnVariables, entityFilters, filtersWithoutEntities, + numResultsPerPage, + setNumResultsPerPage, + isSelectMode, + selectedEntities, + setIsSelectMode, + setSelectedEntities, + onChangeSelectAll, + refetch, }: Props) => { const pageStart = searchResponse?.start || 0; const pageSize = searchResponse?.count || 0; const totalResults = searchResponse?.total || 0; const lastResultIndex = pageStart + pageSize > totalResults ? totalResults : pageStart + pageSize; - - const entityRegistry = useEntityRegistry(); const authenticatedUserUrn = useGetAuthenticatedUser()?.corpUser?.urn; + const combinedSiblingSearchResults = combineSiblingsInSearchResults(searchResponse?.searchResults); - const onResultClick = (result: SearchResult, index: number) => { - analytics.event({ - type: EventType.SearchResultClickEvent, - query, - entityUrn: result.entity.urn, - entityType: result.entity.type, - index, - total: totalResults, - }); - }; - - const onFilterSelect = (newFilters) => { - onChangeFilters(newFilters); - }; - - const history = useHistory(); + const searchResultUrns = combinedSiblingSearchResults.map((result) => result.entity.urn) || []; + const selectedEntityUrns = selectedEntities.map((entity) => entity.urn); return ( <> @@ -183,70 +165,65 @@ export const SearchResults = ({ loading={loading} facets={filters || []} selectedFilters={selectedFilters} - onFilterSelect={onFilterSelect} + onFilterSelect={(newFilters) => onChangeFilters(newFilters)} /> - - Showing{' '} - - {lastResultIndex > 0 ? (page - 1) * pageSize + 1 : 0} - {lastResultIndex} - {' '} - of {totalResults} results - - - - + <> + + Showing{' '} + + {lastResultIndex > 0 ? (page - 1) * pageSize + 1 : 0} - {lastResultIndex} + {' '} + of {totalResults} results + + + + + + {isSelectMode && ( + + 0 && + isListSubset(searchResultUrns, selectedEntityUrns) + } + selectedEntities={selectedEntities} + onChangeSelectAll={onChangeSelectAll} + onCancel={() => setIsSelectMode(false)} + refetch={refetch} + /> + + )} {!loading && ( <> - >> - dataSource={searchResponse?.searchResults} - split={false} - locale={{ - emptyText: ( - - - - - ), - }} - renderItem={(item, index) => ( - <> - onResultClick(item, index)} - > - {entityRegistry.renderSearchResult(item.entity.type, item)} - - - - )} + SearchCfg.RESULTS_PER_PAGE} + onShowSizeChange={(_currNum, newNum) => setNumResultsPerPage(newNum)} + pageSizeOptions={['10', '20', '50']} /> {authenticatedUserUrn && ( diff --git a/datahub-web-react/src/app/search/SearchablePage.tsx b/datahub-web-react/src/app/search/SearchablePage.tsx index 0d79b75339ecba..91af44df04a999 100644 --- a/datahub-web-react/src/app/search/SearchablePage.tsx +++ b/datahub-web-react/src/app/search/SearchablePage.tsx @@ -1,11 +1,14 @@ -import React, { useEffect } from 'react'; -import { useHistory } from 'react-router'; +import React, { useEffect, useState } from 'react'; +import { useHistory, useLocation } from 'react-router'; +import * as QueryString from 'query-string'; import { useTheme } from 'styled-components'; - import { SearchHeader } from './SearchHeader'; import { useEntityRegistry } from '../useEntityRegistry'; import { EntityType } from '../../types.generated'; -import { useGetAutoCompleteMultipleResultsLazyQuery } from '../../graphql/search.generated'; +import { + GetAutoCompleteMultipleResultsQuery, + useGetAutoCompleteMultipleResultsLazyQuery, +} from '../../graphql/search.generated'; import { navigateToSearchUrl } from './utils/navigateToSearchUrl'; import { useGetAuthenticatedUser } from '../useGetAuthenticatedUser'; import analytics, { EventType } from '../analytics'; @@ -21,13 +24,11 @@ const styles = { }; interface Props extends React.PropsWithChildren { - initialQuery?: string; onSearch?: (query: string, type?: EntityType) => void; onAutoComplete?: (query: string) => void; } const defaultProps = { - initialQuery: '', onSearch: undefined, onAutoComplete: undefined, }; @@ -35,13 +36,24 @@ const defaultProps = { /** * A page that includes a sticky search header (nav bar) */ -export const SearchablePage = ({ initialQuery, onSearch, onAutoComplete, children }: Props) => { +export const SearchablePage = ({ onSearch, onAutoComplete, children }: Props) => { + const location = useLocation(); + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const currentQuery: string = decodeURIComponent(params.query ? (params.query as string) : ''); + const history = useHistory(); const entityRegistry = useEntityRegistry(); const themeConfig = useTheme(); const [getAutoCompleteResults, { data: suggestionsData }] = useGetAutoCompleteMultipleResultsLazyQuery(); const user = useGetAuthenticatedUser()?.corpUser; + const [newSuggestionData, setNewSuggestionData] = useState(); + + useEffect(() => { + if (suggestionsData !== undefined) { + setNewSuggestionData(suggestionsData); + } + }, [suggestionsData]); const search = (query: string, type?: EntityType) => { if (!query || query.trim().length === 0) { @@ -75,26 +87,26 @@ export const SearchablePage = ({ initialQuery, onSearch, onAutoComplete, childre // Load correct autocomplete results on initial page load. useEffect(() => { - if (initialQuery && initialQuery.trim() !== '') { + if (currentQuery && currentQuery.trim() !== '') { getAutoCompleteResults({ variables: { input: { - query: initialQuery, + query: currentQuery, }, }, }); } - }, [initialQuery, getAutoCompleteResults]); + }, [currentQuery, getAutoCompleteResults]); return ( <> { data: tokensData, refetch: tokensRefetch, } = useListAccessTokensQuery({ + skip: !canGeneratePersonalAccessTokens, variables: { input: { start, diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx index 0d359c2305f584..8d2e0ab32b319b 100644 --- a/datahub-web-react/src/app/settings/SettingsPage.tsx +++ b/datahub-web-react/src/app/settings/SettingsPage.tsx @@ -6,7 +6,6 @@ import styled from 'styled-components'; import { ANTD_GRAY } from '../entity/shared/constants'; import { ManageIdentities } from '../identity/ManageIdentities'; import { ManagePolicies } from '../policy/ManagePolicies'; -import { SearchablePage } from '../search/SearchablePage'; import { useAppConfig } from '../useAppConfig'; import { useGetAuthenticatedUser } from '../useGetAuthenticatedUser'; import { AccessTokens } from './AccessTokens'; @@ -17,7 +16,7 @@ const PageContainer = styled.div` const SettingsBarContainer = styled.div` padding-top: 20px; - height: 100vh; + min-height: 100vh; border-right: 1px solid ${ANTD_GRAY[5]}; `; @@ -78,56 +77,54 @@ export const SettingsPage = () => { const showUsersGroups = (isIdentityManagementEnabled && me && me.platformPrivileges.manageIdentities) || false; return ( - - - - - Settings - Manage your DataHub settings. - - - { - history.push(`${url}/${newPath.key}`); - }} - > - - - - Access Tokens - + + + + Settings + Manage your DataHub settings. + + + { + history.push(`${url}/${newPath.key}`); + }} + > + + + + Access Tokens + + + {(showPolicies || showUsersGroups) && ( + + {showUsersGroups && ( + + + Users & Groups + + )} + {showPolicies && ( + + + Privileges + + )} - {(showPolicies || showUsersGroups) && ( - - {showPolicies && ( - - - Users & Groups - - )} - {showUsersGroups && ( - - - Privileges - - )} - - )} - - - - - - - {PATHS.map((p) => ( - p.content} key={p.path} /> - ))} - - - + )} + + + + + + + {PATHS.map((p) => ( + p.content} key={p.path} /> + ))} + + ); }; diff --git a/datahub-web-react/src/app/shared/DomainLabel.tsx b/datahub-web-react/src/app/shared/DomainLabel.tsx new file mode 100644 index 00000000000000..40208026d4369f --- /dev/null +++ b/datahub-web-react/src/app/shared/DomainLabel.tsx @@ -0,0 +1,29 @@ +import React from 'react'; +import styled from 'styled-components'; + +const DomainContainerWrapper = styled.div` + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px; +`; + +const DomainContentWrapper = styled.div` + display: flex; + justify-content: center; + align-items: center; +`; + +type Props = { + name: string; +}; + +export const DomainLabel = ({ name }: Props) => { + return ( + + +
{name}
+ + + ); +}; diff --git a/datahub-web-react/src/app/shared/ManageAccount.tsx b/datahub-web-react/src/app/shared/ManageAccount.tsx index 36a57e297c0ccd..b4f18300e897e9 100644 --- a/datahub-web-react/src/app/shared/ManageAccount.tsx +++ b/datahub-web-react/src/app/shared/ManageAccount.tsx @@ -2,7 +2,6 @@ import React from 'react'; import Cookies from 'js-cookie'; import { Menu, Dropdown } from 'antd'; import { CaretDownOutlined } from '@ant-design/icons'; -import { Link } from 'react-router-dom'; import styled, { useTheme } from 'styled-components'; import { EntityType } from '../../types.generated'; import { useEntityRegistry } from '../useEntityRegistry'; @@ -14,6 +13,9 @@ import { ANTD_GRAY } from '../entity/shared/constants'; import { useAppConfig } from '../useAppConfig'; const MenuItem = styled(Menu.Item)` + display: flex; + justify-content: start; + align-items: center; && { margin-top: 2px; } @@ -32,8 +34,10 @@ const DownArrow = styled(CaretDownOutlined)` color: ${ANTD_GRAY[7]}; `; -const StyledLink = styled(Link)` - white-space: nowrap; +const DropdownWrapper = styled.div` + align-items: center; + cursor: pointer; + display: flex; `; interface Props { @@ -57,8 +61,23 @@ export const ManageAccount = ({ urn: _urn, pictureLink: _pictureLink, name }: Pr }; const version = config?.appVersion; const menu = ( - - {version && {version}} + + {version && ( + + {version} + + )} + + + + Your Profile + + + {themeConfig.content.menu.items.map((value) => { return ( @@ -79,20 +98,21 @@ export const ManageAccount = ({ urn: _urn, pictureLink: _pictureLink, name }: Pr OpenAPI + - Logout + Sign Out ); return ( - - + + - + ); }; diff --git a/datahub-web-react/src/app/shared/NoPageFound.tsx b/datahub-web-react/src/app/shared/NoPageFound.tsx index d7b179e06ba274..897af56c6223e9 100644 --- a/datahub-web-react/src/app/shared/NoPageFound.tsx +++ b/datahub-web-react/src/app/shared/NoPageFound.tsx @@ -1,19 +1,74 @@ import React from 'react'; -import { useReactiveVar } from '@apollo/client'; -import { Redirect } from 'react-router'; -import { isLoggedInVar } from '../auth/checkAuthStatus'; -import { PageRoutes } from '../../conf/Global'; +import { Button } from 'antd'; +import styled from 'styled-components'; +import { useHistory } from 'react-router'; + +const MainContainer = styled.div` + height: 100vh; + display: flex; + justify-content: center; + align-items: center; +`; + +const PageNotFoundContainer = styled.div` + max-width: 520px; + width: 100%; + line-height: 1.4; + text-align: center; +`; + +const PageNotFoundTextContainer = styled.div` + position: relative; + height: 240px; +`; + +const NumberContainer = styled.h1` + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + font-size: 252px; + font-weight: 900; + margin: 0px; + color: #262626; + text-transform: uppercase; + letter-spacing: -40px; + margin-left: -20px; +`; + +const Number = styled.span` + text-shadow: -8px 0px 0px #fff; +`; + +const SubTitle = styled.h2` + font-size: 20px; + font-weight: 400; + text-transform: uppercase; + color: #000; + margin-top: 0px; + margin-bottom: 25px; +`; export const NoPageFound = () => { - const isLoggedIn = useReactiveVar(isLoggedInVar); + const history = useHistory(); - if (!isLoggedIn) { - return ; - } + const goToHomepage = () => { + history.push('/'); + }; return ( -
-

Page Not Found!

-
+ + + + + 4 + 0 + 4 + + + The page your requested was not found, + + + ); }; diff --git a/datahub-web-react/src/app/shared/OwnerLabel.tsx b/datahub-web-react/src/app/shared/OwnerLabel.tsx new file mode 100644 index 00000000000000..de3c03dea2ba4a --- /dev/null +++ b/datahub-web-react/src/app/shared/OwnerLabel.tsx @@ -0,0 +1,34 @@ +import React from 'react'; +import styled from 'styled-components'; +import { EntityType } from '../../types.generated'; +import { CustomAvatar } from './avatar'; + +const OwnerContainerWrapper = styled.div` + display: flex; + justify-content: space-between; + align-items: center; + padding: 2px; +`; + +const OwnerContentWrapper = styled.div` + display: flex; + justify-content: center; + align-items: center; +`; + +type Props = { + name: string; + avatarUrl: string | undefined; + type: EntityType; +}; + +export const OwnerLabel = ({ name, avatarUrl, type }: Props) => { + return ( + + + +
{name}
+ + + ); +}; diff --git a/datahub-web-react/src/app/shared/TagStyleEntity.tsx b/datahub-web-react/src/app/shared/TagStyleEntity.tsx index 23460ccef8dc9a..4ea6be06d263f7 100644 --- a/datahub-web-react/src/app/shared/TagStyleEntity.tsx +++ b/datahub-web-react/src/app/shared/TagStyleEntity.tsx @@ -17,7 +17,7 @@ import { useUpdateDescriptionMutation, useSetTagColorMutation } from '../../grap import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated'; import analytics, { EventType, EntityActionType } from '../analytics'; import { GetSearchResultsParams, SearchResultInterface } from '../entity/shared/components/styled/search/types'; -import { AddOwnersModal } from '../entity/shared/containers/profile/sidebar/Ownership/AddOwnersModal'; +import { EditOwnersModal } from '../entity/shared/containers/profile/sidebar/Ownership/EditOwnersModal'; import CopyUrn from './CopyUrn'; import EntityDropdown from '../entity/shared/EntityDropdown'; import { EntityMenuItems } from '../entity/shared/EntityDropdown/EntityDropdown'; @@ -330,7 +330,12 @@ export default function TagStyleEntity({ urn, useGetSearchResults = useWrappedSe setCopiedUrn(true)} /> - + {displayColorPicker && ( @@ -414,16 +419,17 @@ export default function TagStyleEntity({ urn, useGetSearchResults = useWrappedSe
- { - setShowAddModal(false); - }} - urn={urn} - type={EntityType.Tag} - /> + {showAddModal && ( + { + setShowAddModal(false); + }} + urns={[urn]} + entityType={EntityType.Tag} + /> + )}
diff --git a/datahub-web-react/src/app/shared/admin/AdminHeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx similarity index 56% rename from datahub-web-react/src/app/shared/admin/AdminHeaderLinks.tsx rename to datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index b931a78b1e30fb..34ae20e1f8a2dd 100644 --- a/datahub-web-react/src/app/shared/admin/AdminHeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -14,7 +14,7 @@ import { Button, Dropdown, Menu } from 'antd'; import { useAppConfig } from '../../useAppConfig'; import { useGetAuthenticatedUser } from '../../useGetAuthenticatedUser'; -const AdminLink = styled.span` +const LinkWrapper = styled.span` margin-right: 0px; `; @@ -40,7 +40,7 @@ interface Props { areLinksHidden?: boolean; } -export function AdminHeaderLinks(props: Props) { +export function HeaderLinks(props: Props) { const { areLinksHidden } = props; const me = useGetAuthenticatedUser(); const { config } = useAppConfig(); @@ -52,66 +52,58 @@ export function AdminHeaderLinks(props: Props) { const showSettings = true; const showIngestion = isIngestionEnabled && me && me.platformPrivileges.manageIngestion && me.platformPrivileges.manageSecrets; - const showDomains = me?.platformPrivileges?.manageDomains || false; - const showGlossary = me?.platformPrivileges?.manageGlossaries || false; return ( {showAnalytics && ( - + - + )} {showIngestion && ( - + - - )} - {(showGlossary || showDomains) && ( - - {showGlossary && ( - - - Glossary - - - )} - {showDomains && ( - - - Domains - - - )} - - } - > - - - - + )} + + + + Glossary + + + + + Domains + + + + } + > + + + + {showSettings && ( - + - + )} ); diff --git a/datahub-web-react/src/app/shared/deleteUtils.ts b/datahub-web-react/src/app/shared/deleteUtils.ts new file mode 100644 index 00000000000000..a1ff5c845172c2 --- /dev/null +++ b/datahub-web-react/src/app/shared/deleteUtils.ts @@ -0,0 +1,56 @@ +import { useDeleteAssertionMutation } from '../../graphql/assertion.generated'; +import { useDeleteDomainMutation } from '../../graphql/domain.generated'; +import { useDeleteGlossaryEntityMutation } from '../../graphql/glossary.generated'; +import { useRemoveGroupMutation } from '../../graphql/group.generated'; +import { useDeleteTagMutation } from '../../graphql/tag.generated'; +import { useRemoveUserMutation } from '../../graphql/user.generated'; +import { EntityType } from '../../types.generated'; + +/** + * Returns a relative redirect path which is used after an Entity has been deleted from it's profile page. + * + * @param type the entity type being deleted + */ +export const getEntityProfileDeleteRedirectPath = (type: EntityType) => { + switch (type) { + case EntityType.CorpGroup: + case EntityType.CorpUser: + case EntityType.Domain: + case EntityType.Tag: + // Return Home. + return '/'; + case EntityType.GlossaryNode: + case EntityType.GlossaryTerm: + // Return to glossary page. + return '/glossary'; + default: + return () => undefined; + } +}; + +/** + * Returns a mutation hook for deleting an entity of a particular type. + * + * TODO: Push this back into the entity registry. + * + * @param type the entity type being deleted + */ +export const getDeleteEntityMutation = (type: EntityType) => { + switch (type) { + case EntityType.CorpGroup: + return useRemoveGroupMutation; + case EntityType.CorpUser: + return useRemoveUserMutation; + case EntityType.Assertion: + return useDeleteAssertionMutation; + case EntityType.Domain: + return useDeleteDomainMutation; + case EntityType.Tag: + return useDeleteTagMutation; + case EntityType.GlossaryNode: + case EntityType.GlossaryTerm: + return useDeleteGlossaryEntityMutation; + default: + return () => undefined; + } +}; diff --git a/datahub-web-react/src/app/shared/formatNumber.ts b/datahub-web-react/src/app/shared/formatNumber.ts index 55a1a440637e56..e8535d146cb633 100644 --- a/datahub-web-react/src/app/shared/formatNumber.ts +++ b/datahub-web-react/src/app/shared/formatNumber.ts @@ -1,7 +1,11 @@ export function formatNumber(n) { if (n < 1e3) return n; - if (n >= 1e3 && n < 1e6) return `${+(n / 1e3).toFixed(1)}K`; + if (n >= 1e3 && n < 1e6) return `${+(n / 1e3).toFixed(1)}k`; if (n >= 1e6 && n < 1e9) return `${+(n / 1e6).toFixed(1)}M`; if (n >= 1e9) return `${+(n / 1e9).toFixed(1)}B`; return ''; } + +export function formatNumberWithoutAbbreviation(n) { + return n.toLocaleString(); +} diff --git a/datahub-web-react/src/app/shared/health/healthUtils.tsx b/datahub-web-react/src/app/shared/health/healthUtils.tsx index 4278395a96b084..e43abee54fd726 100644 --- a/datahub-web-react/src/app/shared/health/healthUtils.tsx +++ b/datahub-web-react/src/app/shared/health/healthUtils.tsx @@ -1,6 +1,6 @@ import { CheckOutlined, CloseOutlined, WarningOutlined } from '@ant-design/icons'; import React from 'react'; -import { HealthStatus } from '../../../types.generated'; +import { HealthStatus, HealthStatusType } from '../../../types.generated'; export const getHealthColor = (status: HealthStatus) => { switch (status) { @@ -18,7 +18,7 @@ export const getHealthColor = (status: HealthStatus) => { } }; -export const getHealthIcon = (status: HealthStatus, fontSize: number) => { +export const getAssertionsHealthIcon = (status: HealthStatus, fontSize: number) => { switch (status) { case HealthStatus.Pass: { return ; @@ -33,3 +33,13 @@ export const getHealthIcon = (status: HealthStatus, fontSize: number) => { throw new Error(`Unrecognized Health Status ${status} provided`); } }; + +export const getHealthIcon = (type: HealthStatusType, status: HealthStatus, fontSize: number) => { + switch (type) { + case HealthStatusType.Assertions: { + return getAssertionsHealthIcon(status, fontSize); + } + default: + throw new Error(`Unrecognized Health Status Type ${type} provided`); + } +}; diff --git a/datahub-web-react/src/app/shared/recommendation.tsx b/datahub-web-react/src/app/shared/recommendation.tsx new file mode 100644 index 00000000000000..19a30877961526 --- /dev/null +++ b/datahub-web-react/src/app/shared/recommendation.tsx @@ -0,0 +1,18 @@ +import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated'; +import { EntityType } from '../../types.generated'; + +export const useGetRecommendations = (types: Array) => { + const { data } = useGetSearchResultsForMultipleQuery({ + variables: { + input: { + types, + query: '*', + start: 0, + count: 5, + }, + }, + }); + + const recommendedData = data?.searchAcrossEntities?.searchResults?.map((searchResult) => searchResult.entity) || []; + return [recommendedData]; +}; diff --git a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx index 00479f3a0514e9..6a949bee344017 100644 --- a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx +++ b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx @@ -1,32 +1,55 @@ -import React, { useState } from 'react'; +import React, { useRef, useState } from 'react'; import { message, Button, Modal, Select, Typography, Tag as CustomTag } from 'antd'; import styled from 'styled-components'; import { useGetSearchResultsLazyQuery } from '../../../graphql/search.generated'; -import { EntityType, SubResourceType, SearchResult, Tag } from '../../../types.generated'; +import { EntityType, Tag, Entity, ResourceRefInput } from '../../../types.generated'; import CreateTagModal from './CreateTagModal'; -import { useEntityRegistry } from '../../useEntityRegistry'; -import { useAddTagsMutation, useAddTermsMutation } from '../../../graphql/mutations.generated'; -import analytics, { EventType, EntityActionType } from '../../analytics'; +import { + useBatchAddTagsMutation, + useBatchAddTermsMutation, + useBatchRemoveTagsMutation, + useBatchRemoveTermsMutation, +} from '../../../graphql/mutations.generated'; import { useEnterKeyListener } from '../useEnterKeyListener'; import TermLabel from '../TermLabel'; import TagLabel from '../TagLabel'; import GlossaryBrowser from '../../glossary/GlossaryBrowser/GlossaryBrowser'; import ClickOutside from '../ClickOutside'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { useGetRecommendations } from '../recommendation'; +import { FORBIDDEN_URN_CHARS_REGEX } from '../../entity/shared/utils'; + +export enum OperationType { + ADD, + REMOVE, +} -type AddTagsModalProps = { +type EditTagsModalProps = { visible: boolean; onCloseModal: () => void; - entityUrn: string; + resources: ResourceRefInput[]; + // eslint-disable-next-line entityType: EntityType; - entitySubresource?: string; type?: EntityType; + operationType?: OperationType; }; const TagSelect = styled(Select)` width: 480px; `; +const StyleTag = styled(CustomTag)` + margin-right: 3px; + display: flex; + justify-content: start; + align-items: center; + white-space: nowrap; + opacity: 1; + color: #434343; + line-height: 16px; +`; + export const BrowserWrapper = styled.div<{ isHidden: boolean }>` background-color: white; border-radius: 5px; @@ -47,27 +70,35 @@ export const BrowserWrapper = styled.div<{ isHidden: boolean }>` const CREATE_TAG_VALUE = '____reserved____.createTagValue'; -export default function AddTagsTermsModal({ +const isValidTagName = (tagName: string) => { + return tagName && tagName.length > 0 && !FORBIDDEN_URN_CHARS_REGEX.test(tagName); +}; + +export default function EditTagTermsModal({ visible, onCloseModal, - entityUrn, - entityType, - entitySubresource, + resources, type = EntityType.Tag, -}: AddTagsModalProps) { + operationType = OperationType.ADD, +}: EditTagsModalProps) { const entityRegistry = useEntityRegistry(); const [inputValue, setInputValue] = useState(''); const [showCreateModal, setShowCreateModal] = useState(false); - const [disableAdd, setDisableAdd] = useState(false); + const [disableAction, setDisableAction] = useState(false); const [urns, setUrns] = useState([]); const [selectedTerms, setSelectedTerms] = useState([]); + const [selectedTags, setSelectedTags] = useState([]); const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); - const [addTagsMutation] = useAddTagsMutation(); - const [addTermsMutation] = useAddTermsMutation(); + const [batchAddTagsMutation] = useBatchAddTagsMutation(); + const [batchRemoveTagsMutation] = useBatchRemoveTagsMutation(); + const [batchAddTermsMutation] = useBatchAddTermsMutation(); + const [batchRemoveTermsMutation] = useBatchRemoveTermsMutation(); const [tagTermSearch, { data: tagTermSearchData }] = useGetSearchResultsLazyQuery(); - const tagSearchResults = tagTermSearchData?.search?.searchResults || []; + const tagSearchResults = tagTermSearchData?.search?.searchResults?.map((searchResult) => searchResult.entity) || []; + const [recommendedData] = useGetRecommendations([EntityType.Tag]); + const inputEl = useRef(null); const handleSearch = (text: string) => { if (text.length > 0) { @@ -84,39 +115,46 @@ export default function AddTagsTermsModal({ } }; - const renderSearchResult = (result: SearchResult) => { + const renderSearchResult = (entity: Entity) => { const displayName = - result.entity.type === EntityType.Tag - ? (result.entity as Tag).name - : entityRegistry.getDisplayName(result.entity.type, result.entity); + entity.type === EntityType.Tag ? (entity as Tag).name : entityRegistry.getDisplayName(entity.type, entity); const tagOrTermComponent = - result.entity.type === EntityType.Tag ? ( + entity.type === EntityType.Tag ? ( ) : ( ); return ( - + {tagOrTermComponent} ); }; - const tagSearchOptions = tagSearchResults.map((result) => { + const tagResult = + (!inputValue || inputValue.length === 0) && type === EntityType.Tag ? recommendedData : tagSearchResults; + + const tagSearchOptions = tagResult?.map((result) => { return renderSearchResult(result); }); - const inputExistsInTagSearch = tagSearchResults.some((result: SearchResult) => { - const displayName = entityRegistry.getDisplayName(result.entity.type, result.entity); + const inputExistsInTagSearch = tagSearchResults.some((entity: Entity) => { + const displayName = entityRegistry.getDisplayName(entity.type, entity); return displayName.toLowerCase() === inputValue.toLowerCase(); }); - if (!inputExistsInTagSearch && inputValue.length > 0 && type === EntityType.Tag && urns.length === 0) { - tagSearchOptions.push( + if ( + operationType === OperationType.ADD && + !inputExistsInTagSearch && + isValidTagName(inputValue) && + type === EntityType.Tag && + urns.length === 0 + ) { + tagSearchOptions?.push( Create {inputValue} , @@ -125,32 +163,20 @@ export default function AddTagsTermsModal({ const tagRender = (props) => { // eslint-disable-next-line react/prop-types - const { label, closable, onClose, value } = props; + const { closable, onClose, value } = props; const onPreventMouseDown = (event) => { event.preventDefault(); event.stopPropagation(); }; const selectedItem = - type === EntityType.GlossaryTerm ? selectedTerms.find((term) => term.urn === value).component : label; + type === EntityType.GlossaryTerm + ? selectedTerms.find((term) => term.urn === value).component + : selectedTags.find((term) => term.urn === value).component; return ( - + {selectedItem} - + ); }; @@ -166,8 +192,7 @@ export default function AddTagsTermsModal({ onClose={onCloseModal} onBack={() => setShowCreateModal(false)} tagName={inputValue} - entityUrn={entityUrn} - entitySubresource={entitySubresource} + resources={resources} /> ); } @@ -175,13 +200,32 @@ export default function AddTagsTermsModal({ // When a Tag or term search result is selected, add the urn to the Urns const onSelectValue = (urn: string) => { if (urn === CREATE_TAG_VALUE) { - setShowCreateModal(true); + if (isValidTagName(inputValue)) { + setShowCreateModal(true); + } return; } const newUrns = [...(urns || []), urn]; + const selectedSearchOption = tagSearchOptions?.find((option) => option.props.value === urn); + const selectedTagOption = tagResult?.find((tag) => tag.urn === urn); setUrns(newUrns); - const selectedSearchOption = tagSearchOptions.find((option) => option.props.value === urn); setSelectedTerms([...selectedTerms, { urn, component: }]); + setSelectedTags([ + ...selectedTags, + { + urn, + component: ( + + ), + }, + ]); + if (inputEl && inputEl.current) { + (inputEl.current as any).blur(); + } }; // When a Tag or term search result is deselected, remove the urn from the Owners @@ -191,63 +235,44 @@ export default function AddTagsTermsModal({ setInputValue(''); setIsFocusedOnInput(true); setSelectedTerms(selectedTerms.filter((term) => term.urn !== urn)); + setSelectedTags(selectedTags.filter((term) => term.urn !== urn)); }; - // Function to handle the modal action's - const onOk = () => { - let mutation: ((input: any) => Promise) | null = null; - if (type === EntityType.Tag) { - mutation = addTagsMutation; - } - if (type === EntityType.GlossaryTerm) { - mutation = addTermsMutation; - } - - if (!entityUrn || !mutation) { - onCloseModal(); - return; - } - setDisableAdd(true); - - let input = {}; - let actionType = EntityActionType.UpdateSchemaTags; - if (type === EntityType.Tag) { - input = { - tagUrns: urns, - resourceUrn: entityUrn, - subResource: entitySubresource, - subResourceType: entitySubresource ? SubResourceType.DatasetField : null, - }; - if (entitySubresource) { - actionType = EntityActionType.UpdateSchemaTags; - } else { - actionType = EntityActionType.UpdateTags; - } - } - if (type === EntityType.GlossaryTerm) { - input = { - termUrns: urns, - resourceUrn: entityUrn, - subResource: entitySubresource, - subResourceType: entitySubresource ? SubResourceType.DatasetField : null, - }; - if (entitySubresource) { - actionType = EntityActionType.UpdateSchemaTerms; - } else { - actionType = EntityActionType.UpdateTerms; - } - } - - analytics.event({ - type: EventType.EntityActionEvent, - entityType, - entityUrn, - actionType, - }); + const batchAddTags = () => { + batchAddTagsMutation({ + variables: { + input: { + tagUrns: urns, + resources, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ + content: `Added ${type === EntityType.GlossaryTerm ? 'Terms' : 'Tags'}!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to add: \n ${e.message || ''}`, duration: 3 }); + }) + .finally(() => { + setDisableAction(false); + onCloseModal(); + setUrns([]); + }); + }; - mutation({ + const batchAddTerms = () => { + batchAddTermsMutation({ variables: { - input, + input: { + termUrns: urns, + resources, + }, }, }) .then(({ errors }) => { @@ -263,12 +288,99 @@ export default function AddTagsTermsModal({ message.error({ content: `Failed to add: \n ${e.message || ''}`, duration: 3 }); }) .finally(() => { - setDisableAdd(false); + setDisableAction(false); + onCloseModal(); + setUrns([]); + }); + }; + + const batchRemoveTags = () => { + batchRemoveTagsMutation({ + variables: { + input: { + tagUrns: urns, + resources, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ + content: `Removed ${type === EntityType.GlossaryTerm ? 'Terms' : 'Tags'}!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to remove: \n ${e.message || ''}`, duration: 3 }); + }) + .finally(() => { + setDisableAction(false); onCloseModal(); setUrns([]); }); }; + const batchRemoveTerms = () => { + batchRemoveTermsMutation({ + variables: { + input: { + termUrns: urns, + resources, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ + content: `Removed ${type === EntityType.GlossaryTerm ? 'Terms' : 'Tags'}!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to remove: \n ${e.message || ''}`, duration: 3 }); + }) + .finally(() => { + setDisableAction(false); + onCloseModal(); + setUrns([]); + }); + }; + + const editTags = () => { + if (operationType === OperationType.ADD) { + batchAddTags(); + } else { + batchRemoveTags(); + } + }; + + const editTerms = () => { + if (operationType === OperationType.ADD) { + batchAddTerms(); + } else { + batchRemoveTerms(); + } + }; + + // Function to handle the modal action's + const onOk = () => { + if (!resources) { + onCloseModal(); + return; + } + setDisableAction(true); + + if (type === EntityType.Tag) { + editTags(); + } else { + editTerms(); + } + }; + function selectTermFromBrowser(urn: string, displayName: string) { setIsFocusedOnInput(false); const newUrns = [...(urns || []), urn]; @@ -289,11 +401,9 @@ export default function AddTagsTermsModal({ return ( } @@ -313,7 +423,9 @@ export default function AddTagsTermsModal({ setIsFocusedOnInput(false)}> setIsFocusedOnInput(true)} onBlur={handleBlur} - dropdownStyle={isShowingGlossaryBrowser || !inputValue ? { display: 'none' } : {}} + dropdownStyle={isShowingGlossaryBrowser ? { display: 'none' } : {}} > {tagSearchOptions} diff --git a/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx b/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx index 945b7e49762e36..bcc254f10da7b6 100644 --- a/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx +++ b/datahub-web-react/src/app/shared/tags/CreateTagModal.tsx @@ -1,10 +1,9 @@ import React, { useState } from 'react'; import { message, Button, Input, Modal, Space } from 'antd'; import styled from 'styled-components'; - -import { useUpdateTagMutation } from '../../../graphql/tag.generated'; -import { useAddTagMutation } from '../../../graphql/mutations.generated'; -import { SubResourceType } from '../../../types.generated'; +import { useBatchAddTagsMutation } from '../../../graphql/mutations.generated'; +import { useCreateTagMutation } from '../../../graphql/tag.generated'; +import { ResourceRefInput } from '../../../types.generated'; import { useEnterKeyListener } from '../useEnterKeyListener'; type CreateTagModalProps = { @@ -12,37 +11,28 @@ type CreateTagModalProps = { onClose: () => void; onBack: () => void; tagName: string; - entityUrn: string; - entitySubresource?: string; + resources: ResourceRefInput[]; }; const FullWidthSpace = styled(Space)` width: 100%; `; -export default function CreateTagModal({ - onClose, - onBack, - visible, - tagName, - entityUrn, - entitySubresource, -}: CreateTagModalProps) { +export default function CreateTagModal({ onClose, onBack, visible, tagName, resources }: CreateTagModalProps) { const [stagedDescription, setStagedDescription] = useState(''); - const [addTagMutation] = useAddTagMutation(); + const [batchAddTagsMutation] = useBatchAddTagsMutation(); - const [updateTagMutation] = useUpdateTagMutation(); + const [createTagMutation] = useCreateTagMutation(); const [disableCreate, setDisableCreate] = useState(false); const onOk = () => { setDisableCreate(true); // first create the new tag const tagUrn = `urn:li:tag:${tagName}`; - updateTagMutation({ + createTagMutation({ variables: { - urn: tagUrn, input: { - urn: tagUrn, + id: tagName, name: tagName, description: stagedDescription, }, @@ -50,24 +40,28 @@ export default function CreateTagModal({ }) .then(() => { // then apply the tag to the dataset - addTagMutation({ + batchAddTagsMutation({ variables: { input: { - tagUrn, - resourceUrn: entityUrn, - subResource: entitySubresource, - subResourceType: entitySubresource ? SubResourceType.DatasetField : null, + tagUrns: [tagUrn], + resources, }, }, - }).finally(() => { - // and finally close the modal - setDisableCreate(false); - onClose(); - }); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to add tag: \n ${e.message || ''}`, duration: 3 }); + onClose(); + }) + .finally(() => { + // and finally close the modal + setDisableCreate(false); + onClose(); + }); }) .catch((e) => { message.destroy(); - message.error({ content: `Failed to create & add tag: \n ${e.message || ''}`, duration: 3 }); + message.error({ content: `Failed to create tag: \n ${e.message || ''}`, duration: 3 }); onClose(); }); }; diff --git a/datahub-web-react/src/app/shared/tags/TagTermGroup.tsx b/datahub-web-react/src/app/shared/tags/TagTermGroup.tsx index c8de58acd77772..d9a0ad64a8cd01 100644 --- a/datahub-web-react/src/app/shared/tags/TagTermGroup.tsx +++ b/datahub-web-react/src/app/shared/tags/TagTermGroup.tsx @@ -5,20 +5,28 @@ import styled from 'styled-components'; import { BookOutlined, PlusOutlined } from '@ant-design/icons'; import { useEntityRegistry } from '../../useEntityRegistry'; -import { Domain, EntityType, GlobalTags, GlossaryTerms, SubResourceType } from '../../../types.generated'; +import { + Domain, + EntityType, + GlobalTags, + GlossaryTermAssociation, + GlossaryTerms, + SubResourceType, + TagAssociation, +} from '../../../types.generated'; import { StyledTag } from '../../entity/shared/components/styled/StyledTag'; import { EMPTY_MESSAGES, ANTD_GRAY } from '../../entity/shared/constants'; import { useRemoveTagMutation, useRemoveTermMutation } from '../../../graphql/mutations.generated'; import { DomainLink } from './DomainLink'; import { TagProfileDrawer } from './TagProfileDrawer'; -import AddTagsTermsModal from './AddTagsTermsModal'; +import EditTagTermsModal from './AddTagsTermsModal'; type Props = { uneditableTags?: GlobalTags | null; editableTags?: GlobalTags | null; editableGlossaryTerms?: GlossaryTerms | null; uneditableGlossaryTerms?: GlossaryTerms | null; - domain?: Domain | null; + domain?: Domain | undefined | null; canRemove?: boolean; canAddTag?: boolean; canAddTerm?: boolean; @@ -84,19 +92,19 @@ export default function TagTermGroup({ const [tagProfileDrawerVisible, setTagProfileDrawerVisible] = useState(false); const [addTagUrn, setAddTagUrn] = useState(''); - const removeTag = (urnToRemove: string) => { + const removeTag = (tagAssociationToRemove: TagAssociation) => { + const tagToRemove = tagAssociationToRemove.tag; onOpenModal?.(); - const tagToRemove = editableTags?.tags?.find((tag) => tag.tag.urn === urnToRemove); Modal.confirm({ - title: `Do you want to remove ${tagToRemove?.tag.name} tag?`, - content: `Are you sure you want to remove the ${tagToRemove?.tag.name} tag?`, + title: `Do you want to remove ${tagToRemove?.name} tag?`, + content: `Are you sure you want to remove the ${tagToRemove?.name} tag?`, onOk() { - if (entityUrn) { + if (tagAssociationToRemove.associatedUrn || entityUrn) { removeTagMutation({ variables: { input: { - tagUrn: urnToRemove, - resourceUrn: entityUrn, + tagUrn: tagToRemove.urn, + resourceUrn: tagAssociationToRemove.associatedUrn || entityUrn || '', subResource: entitySubresource, subResourceType: entitySubresource ? SubResourceType.DatasetField : null, }, @@ -121,20 +129,19 @@ export default function TagTermGroup({ }); }; - const removeTerm = (urnToRemove: string) => { + const removeTerm = (termToRemove: GlossaryTermAssociation) => { onOpenModal?.(); - const termToRemove = editableGlossaryTerms?.terms?.find((term) => term.term.urn === urnToRemove); const termName = termToRemove && entityRegistry.getDisplayName(termToRemove.term.type, termToRemove.term); Modal.confirm({ title: `Do you want to remove ${termName} term?`, content: `Are you sure you want to remove the ${termName} term?`, onOk() { - if (entityUrn) { + if (termToRemove.associatedUrn || entityUrn) { removeTermMutation({ variables: { input: { - termUrn: urnToRemove, - resourceUrn: entityUrn, + termUrn: termToRemove.term.urn, + resourceUrn: termToRemove.associatedUrn || entityUrn || '', subResource: entitySubresource, subResourceType: entitySubresource ? SubResourceType.DatasetField : null, }, @@ -192,7 +199,7 @@ export default function TagTermGroup({ to={entityRegistry.getEntityUrl(EntityType.GlossaryTerm, term.term.urn)} key={term.term.urn} > - + {entityRegistry.getDisplayName(EntityType.GlossaryTerm, term.term)} @@ -202,10 +209,11 @@ export default function TagTermGroup({ {editableGlossaryTerms?.terms?.map((term) => ( { e.preventDefault(); - removeTerm(term.term.urn); + removeTerm(term); }} > @@ -225,6 +233,7 @@ export default function TagTermGroup({ return ( showTagProfileDrawer(tag?.tag?.urn)} $colorHash={tag?.tag?.urn} $color={tag?.tag?.properties?.colorHex} @@ -249,7 +258,7 @@ export default function TagTermGroup({ closable={canRemove} onClose={(e) => { e.preventDefault(); - removeTag(tag?.tag?.urn); + removeTag(tag); }} > {tag?.tag?.name} @@ -302,7 +311,7 @@ export default function TagTermGroup({ )} {showAddModal && !!entityUrn && !!entityType && ( - { @@ -310,9 +319,14 @@ export default function TagTermGroup({ setShowAddModal(false); refetch?.(); }} - entityUrn={entityUrn} + resources={[ + { + resourceUrn: entityUrn, + subResource: entitySubresource, + subResourceType: entitySubresource ? SubResourceType.DatasetField : null, + }, + ]} entityType={entityType} - entitySubresource={entitySubresource} /> )} diff --git a/datahub-web-react/src/app/shared/textUtil.ts b/datahub-web-react/src/app/shared/textUtil.ts index 2ce838998b92fd..5a258aff36406c 100644 --- a/datahub-web-react/src/app/shared/textUtil.ts +++ b/datahub-web-react/src/app/shared/textUtil.ts @@ -27,3 +27,7 @@ export function groupIdTextValidation(str: string) { if (str.indexOf(':') > 0) return false; return true; } + +export function pluralize(count: number, noun: string, suffix = 's') { + return `${noun}${count !== 1 ? suffix : ''}`; +} diff --git a/datahub-web-react/src/app/useGetAuthenticatedUser.tsx b/datahub-web-react/src/app/useGetAuthenticatedUser.tsx index 7503861a67bba7..64bee6a32c5e3c 100644 --- a/datahub-web-react/src/app/useGetAuthenticatedUser.tsx +++ b/datahub-web-react/src/app/useGetAuthenticatedUser.tsx @@ -10,7 +10,7 @@ export function useGetAuthenticatedUser(skip?: boolean) { if (!userUrn) { throw new Error('Could not find logged in user.'); } - const { data, error } = useGetMeQuery({ skip }); + const { data, error } = useGetMeQuery({ skip, fetchPolicy: 'cache-first' }); if (error) { console.error(`Could not fetch logged in user from cache. + ${error.message}`); } diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts index d6001f57f455bb..1132efee046f5b 100644 --- a/datahub-web-react/src/conf/Global.ts +++ b/datahub-web-react/src/conf/Global.ts @@ -5,8 +5,11 @@ export enum PageRoutes { /** * Server-side authentication route */ + ROOT = '/', AUTHENTICATE = '/authenticate', + SIGN_UP = '/signup', LOG_IN = '/login', + RESET_CREDENTIALS = '/reset', SEARCH_RESULTS = '/search/:type?', SEARCH = '/search', BROWSE = '/browse', diff --git a/datahub-web-react/src/graphql-mock/fixtures/entity/chartEntity.ts b/datahub-web-react/src/graphql-mock/fixtures/entity/chartEntity.ts index 75e6eab07864d3..3fa3f2b2bef4a0 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/entity/chartEntity.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/entity/chartEntity.ts @@ -60,6 +60,7 @@ export const chartEntity = (tool): Chart => { owner: datahubUser, type: OwnershipType.Stakeholder, __typename: 'Owner', + associatedUrn: `urn:li:chart:(${tool},${name})`, }, ], lastModified: { time: 1619717962718, __typename: 'AuditStamp' }, diff --git a/datahub-web-react/src/graphql-mock/fixtures/entity/dashboardEntity.ts b/datahub-web-react/src/graphql-mock/fixtures/entity/dashboardEntity.ts index 6fc703dd549b6f..73e08118739cc1 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/entity/dashboardEntity.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/entity/dashboardEntity.ts @@ -42,6 +42,7 @@ export const dashboardEntity = (tool): Dashboard => { owner: datahubUser, type: OwnershipType.Stakeholder, __typename: 'Owner', + associatedUrn: `urn:li:dashboard:(${tool},${name})`, }, ], lastModified: { time: 1619993818664, __typename: 'AuditStamp' }, @@ -53,6 +54,7 @@ export const dashboardEntity = (tool): Dashboard => { owner: kafkaUser, type: OwnershipType.Stakeholder, __typename: 'Owner', + associatedUrn: `urn:li:dashboard:(${tool},${name})`, }, ], lastModified: { time: 1619993818664, __typename: 'AuditStamp' }, @@ -83,16 +85,19 @@ export const dashboardEntity = (tool): Dashboard => { { owner: datahubUser, type: OwnershipType.Stakeholder, + associatedUrn: `urn:li:dashboard:(${tool},${name})`, __typename: 'Owner', }, { owner: kafkaUser, type: OwnershipType.Developer, + associatedUrn: `urn:li:dashboard:(${tool},${name})`, __typename: 'Owner', }, { owner: lookerUser, type: OwnershipType.Developer, + associatedUrn: `urn:li:dashboard:(${tool},${name})`, __typename: 'Owner', }, ], @@ -103,10 +108,12 @@ export const dashboardEntity = (tool): Dashboard => { tags: [ { tag: generateTag(datahubOwnership), + associatedUrn: `urn:li:dashboard:(${tool},${name})`, __typename: 'TagAssociation', }, { tag: generateTag(kafkaOwnership), + associatedUrn: `urn:li:dashboard:(${tool},${name})`, __typename: 'TagAssociation', }, ], diff --git a/datahub-web-react/src/graphql-mock/fixtures/entity/dataFlowEntity.ts b/datahub-web-react/src/graphql-mock/fixtures/entity/dataFlowEntity.ts index 497ac1d2c24b18..1f5e3d6daa1df6 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/entity/dataFlowEntity.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/entity/dataFlowEntity.ts @@ -60,6 +60,7 @@ export const dataFlowEntity = ({ orchestrator, cluster }: DataFlowEntityArg): Da { owner: datahubUser, type: OwnershipType.Stakeholder, + associatedUrn: `urn:li:dataFlow:(${orchestrator},${flowId},${cluster})`, __typename: 'Owner', }, ], diff --git a/datahub-web-react/src/graphql-mock/fixtures/entity/dataJobEntity.ts b/datahub-web-react/src/graphql-mock/fixtures/entity/dataJobEntity.ts index c241f414fe29f0..8f740d4e2d64f2 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/entity/dataJobEntity.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/entity/dataJobEntity.ts @@ -55,50 +55,126 @@ export const dataJobEntity = (): DataJob => { project: null, externalUrl: 'https://airflow.demo.datahubproject.io/tree?dag_id=datahub_analytics_refresh', customProperties: [ - { key: 'end_date', value: 'None', __typename: 'StringMapEntry' }, - { key: 'orientation', value: "'LR'", __typename: 'StringMapEntry' }, - { key: 'max_active_runs', value: '16', __typename: 'StringMapEntry' }, - { key: 'is_paused_upon_creation', value: 'None', __typename: 'StringMapEntry' }, - { key: 'timezone', value: "'UTC'", __typename: 'StringMapEntry' }, - { key: 'params', value: '{}', __typename: 'StringMapEntry' }, + { + key: 'end_date', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + value: 'None', + __typename: 'CustomPropertiesEntry', + }, + { + key: 'orientation', + value: "'LR'", + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'max_active_runs', + value: '16', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'is_paused_upon_creation', + value: 'None', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'timezone', + value: "'UTC'", + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'params', + value: '{}', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, { key: 'fileloc', value: "'/opt/airflow/dags/repo/airflow/dags/datahub_analytics_refresh.py'", - __typename: 'StringMapEntry', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', }, { key: 'default_args', value: "{: {'owner': 'harshal', 'depends_on_past': False, 'email': ['harshal@acryl.io'], 'email_on_failure': False, 'execution_timeout': {: 300.0, : }}, : }", - __typename: 'StringMapEntry', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'tags', + value: 'None', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: '_access_control', + value: 'None', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'doc_md', + value: 'None', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'dagrun_timeout', + value: 'None', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', }, - { key: 'tags', value: 'None', __typename: 'StringMapEntry' }, - { key: '_access_control', value: 'None', __typename: 'StringMapEntry' }, - { key: 'doc_md', value: 'None', __typename: 'StringMapEntry' }, - { key: 'dagrun_timeout', value: 'None', __typename: 'StringMapEntry' }, { key: '_dag_id', value: "'datahub_analytics_refresh'", - __typename: 'StringMapEntry', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'catchup', + value: 'False', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', }, - { key: 'catchup', value: 'False', __typename: 'StringMapEntry' }, { key: 'schedule_interval', value: "{: 86400.0, : }", - __typename: 'StringMapEntry', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: '_default_view', + value: 'None', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', }, - { key: '_default_view', value: 'None', __typename: 'StringMapEntry' }, { key: '_description', value: "'Refresh snowflake tables for analytics purposes'", - __typename: 'StringMapEntry', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: '_concurrency', + value: '16', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', }, - { key: '_concurrency', value: '16', __typename: 'StringMapEntry' }, { key: 'tasks', value: "[{'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': []}, 'ui_color': '#f0ede4', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'run_data_task', '_inlets': {'auto': False, 'task_ids': [], 'datasets': []}, 'template_fields': ['bash_command', 'env'], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['split_s3_task'], '_task_type': 'BashOperator', '_task_module': 'airflow.operators.bash_operator', 'bash_command': \"echo 'This is where we might run the backup job'\"}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.all_entities', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.user_basic_info', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.user_extra_info', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_ownerships', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_properties', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_schemas', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_schema_extras', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_tags', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_lineages', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_statuses', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.tag_ownerships', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.tag_properties', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataflow_ownerships', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataflow_info', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataflow_tags', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_ownerships', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_info', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_lineages', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_tags', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.chart_info', env='PROD')\", \"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dashboard_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'split_s3_task', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahub-demo-backup.demo.aspects', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['load_all_entities_to_snowflake', 'load_tag_ownerships_to_snowflake', 'load_dataset_tags_to_snowflake', 'load_dataset_properties_to_snowflake', 'load_dataset_schema_extras_to_snowflake', 'load_dataset_schemas_to_snowflake', 'load_datajob_lineages_to_snowflake', 'load_dataflow_ownerships_to_snowflake', 'load_dashboard_info_to_snowflake', 'load_datajob_tags_to_snowflake', 'load_dataset_statuses_to_snowflake', 'load_chart_info_to_snowflake', 'load_user_basic_info_to_snowflake', 'load_user_extra_info_to_snowflake', 'load_datajob_info_to_snowflake', 'load_dataset_lineages_to_snowflake', 'load_dataset_ownerships_to_snowflake', 'load_datajob_ownerships_to_snowflake', 'load_tag_properties_to_snowflake', 'load_dataflow_info_to_snowflake', 'load_dataflow_tags_to_snowflake'], '_task_type': 'S3FileToDirectoryTransform', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': []}, 'ui_color': '#e8f7e4', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'wait_for_load_finish', '_inlets': {'auto': False, 'task_ids': [], 'datasets': []}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['update_generated_latest_dataset_owners', 'update_generated_latest_dataset_info', 'update_generated_dataset_platforms'], '_task_type': 'DummyOperator', '_task_module': 'airflow.operators.dummy_operator'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.all_entities', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_all_entities_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.all_entities', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.user_basic_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_user_basic_info_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.user_basic_info', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.user_extra_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_user_extra_info_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.user_extra_info', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_ownerships', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_ownerships_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_ownerships', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_properties', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_properties_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_properties', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_schemas', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_schemas_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_schemas', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_schema_extras', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_schema_extras_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_schema_extras', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_tags', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_tags_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_tags', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_lineages', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_lineages_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_lineages', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_statuses', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataset_statuses_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataset_statuses', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.tag_ownerships', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_tag_ownerships_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.tag_ownerships', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.tag_properties', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_tag_properties_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.tag_properties', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataflow_ownerships', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataflow_ownerships_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataflow_ownerships', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataflow_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataflow_info_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataflow_info', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dataflow_tags', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dataflow_tags_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dataflow_tags', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.datajob_ownerships', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_datajob_ownerships_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_ownerships', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.datajob_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_datajob_info_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_info', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.datajob_lineages', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_datajob_lineages_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_lineages', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.datajob_tags', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_datajob_tags_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.datajob_tags', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.chart_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_chart_info_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.chart_info', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.dashboard_info', env='PROD')\"]}, 'ui_color': '#fff', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'load_dashboard_info_to_snowflake', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='s3', name='datahubproject-demo-pipelines.entity_aspect_splits.dashboard_info', env='PROD')\"]}, 'template_fields': [], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['wait_for_load_finish'], '_task_type': 'LoadS3IntoSnowflakeOperator', '_task_module': 'unusual_prefix_436311363ce00ecbd709f747db697044ba2913d4_datahub_analytics_refresh'}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.generated_dataset_platforms', env='PROD')\"]}, 'ui_color': '#ededed', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'update_generated_dataset_platforms', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.all_entities', env='PROD')\"]}, 'template_fields': ['sql'], 'email': ['harshal@acryl.io'], '_downstream_task_ids': ['update_generated_latest_dataset_owners', 'update_generated_latest_dataset_info'], '_task_type': 'SnowflakeOperator', '_task_module': 'airflow.providers.snowflake.operators.snowflake', 'sql': \"\\nCREATE OR REPLACE TABLE generated_dataset_platforms AS (\\n SELECT urn, split(split(urn, ',')[0], ':')[6]::string as platform\\n FROM all_entities\\n WHERE all_entities.entity = 'dataset'\\n);\\n \"}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.generated_latest_dataset_owners', env='PROD')\"]}, 'ui_color': '#ededed', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'update_generated_latest_dataset_owners', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.all_entities', env='PROD')\", \"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_ownerships', env='PROD')\", \"Dataset(platform='snowflake', name='demo_pipeline.public.generated_dataset_platforms', env='PROD')\"]}, 'template_fields': ['sql'], 'email': ['harshal@acryl.io'], '_downstream_task_ids': [], '_task_type': 'SnowflakeOperator', '_task_module': 'airflow.providers.snowflake.operators.snowflake', 'sql': \"\\nCREATE OR REPLACE TABLE generated_latest_dataset_owners AS (\\n WITH latest_dataset_owners AS (SELECT * FROM dataset_ownerships WHERE version = 0)\\n SELECT all_entities.urn, generated_dataset_platforms.platform, metadata:owners as owners, ARRAY_SIZE(COALESCE(metadata:owners, array_construct())) as owner_count\\n FROM all_entities\\n LEFT JOIN latest_dataset_owners ON all_entities.urn = latest_dataset_owners.urn\\n LEFT JOIN generated_dataset_platforms ON all_entities.urn = generated_dataset_platforms.urn\\n WHERE all_entities.entity = 'dataset'\\n);\\n \"}, {'execution_timeout': 300.0, 'ui_fgcolor': '#000', '_outlets': {'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.generated_latest_dataset_info', env='PROD')\"]}, 'ui_color': '#ededed', 'email_on_failure': False, 'owner': 'harshal', 'task_id': 'update_generated_latest_dataset_info', '_inlets': {'auto': False, 'task_ids': [], 'datasets': [\"Dataset(platform='snowflake', name='demo_pipeline.public.all_entities', env='PROD')\", \"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_properties', env='PROD')\", \"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_tags', env='PROD')\", \"Dataset(platform='snowflake', name='demo_pipeline.public.dataset_lineages', env='PROD')\", \"Dataset(platform='snowflake', name='demo_pipeline.public.generated_dataset_platforms', env='PROD')\"]}, 'template_fields': ['sql'], 'email': ['harshal@acryl.io'], '_downstream_task_ids': [], '_task_type': 'SnowflakeOperator', '_task_module': 'airflow.providers.snowflake.operators.snowflake', 'sql': \"\\nCREATE OR REPLACE TABLE generated_latest_dataset_info AS (\\n WITH\\n latest_dataset_info AS (SELECT * FROM dataset_properties WHERE version = 0),\\n latest_dataset_tags AS (SELECT * FROM dataset_tags WHERE version = 0),\\n latest_dataset_lineages AS (SELECT * FROM dataset_lineages WHERE version = 0)\\n SELECT\\n all_entities.urn,\\n generated_dataset_platforms.platform,\\n latest_dataset_info.metadata:description::string as description,\\n COALESCE(IS_NULL_VALUE(latest_dataset_info.metadata:description), TRUE) as is_missing_docs,\\n latest_dataset_info.metadata:customProperties as properties,\\n ARRAY_CAT(COALESCE(latest_dataset_tags.metadata:tags, array_construct()), COALESCE(latest_dataset_info.metadata:tags, array_construct())) as tags,\\n latest_dataset_lineages.metadata:upstreams as upstreams,\\n COALESCE(ARRAY_SIZE(latest_dataset_lineages.metadata:upstreams), 0) as upstream_count,\\n COALESCE(ARRAY_SIZE(latest_dataset_lineages.metadata:upstreams), 0) > 0 as has_upstreams\\n FROM all_entities\\n LEFT JOIN latest_dataset_info ON all_entities.urn = latest_dataset_info.urn\\n LEFT JOIN latest_dataset_tags ON all_entities.urn = latest_dataset_tags.urn\\n LEFT JOIN latest_dataset_lineages ON all_entities.urn = latest_dataset_lineages.urn\\n LEFT JOIN generated_dataset_platforms ON all_entities.urn = generated_dataset_platforms.urn\\n WHERE all_entities.entity = 'dataset'\\n);\\n \"}]", - __typename: 'StringMapEntry', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', + }, + { + key: 'start_date', + value: '1619913600.0', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, + __typename: 'CustomPropertiesEntry', }, - { key: 'start_date', value: '1619913600.0', __typename: 'StringMapEntry' }, ], __typename: 'DataFlowInfo', }, @@ -118,6 +194,7 @@ export const dataJobEntity = (): DataJob => { owner: kafkaUser, type: OwnershipType.Developer, __typename: 'Owner', + associatedUrn: `urn:li:dataJob:${dataFlowURN},${jobId})`, }, ], lastModified: { time: 1620079975489, __typename: 'AuditStamp' }, diff --git a/datahub-web-react/src/graphql-mock/fixtures/entity/datasetEntity.ts b/datahub-web-react/src/graphql-mock/fixtures/entity/datasetEntity.ts index b7d90561f393fd..d7476f6c44a162 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/entity/datasetEntity.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/entity/datasetEntity.ts @@ -35,11 +35,7 @@ export type DatasetEntityArg = { path: string; }; -export const datasetEntity = ({ - platform, - origin, - path, -}: DatasetEntityArg): Dataset & { previousSchemaMetadata: any } => { +export const datasetEntity = ({ platform, origin, path }: DatasetEntityArg): Dataset => { const name = `${path}.${faker.company.bsNoun()}_${faker.company.bsNoun()}`; const description = `${faker.commerce.productDescription()}`; const datahubUser = findUserByUsername('datahub'); @@ -63,6 +59,7 @@ export const datasetEntity = ({ { owner: datahubUser, type: OwnershipType.Dataowner, + associatedUrn: `urn:li:dataset:(${platformURN},${name},${origin.toUpperCase()})`, __typename: 'Owner', }, ], @@ -80,8 +77,8 @@ export const datasetEntity = ({ usageStats: null, glossaryTerms: null, schemaMetadata: null, - previousSchemaMetadata: null, __typename: 'Dataset', subTypes: null, + health: [], }; }; diff --git a/datahub-web-react/src/graphql-mock/fixtures/tag.ts b/datahub-web-react/src/graphql-mock/fixtures/tag.ts index 1c5e01d26521c4..a681e421b1b8a5 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/tag.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/tag.ts @@ -34,6 +34,7 @@ export const createTag = ({ name, urn, description }: TagUpdateInput): Tag => { { owner: user, type: OwnershipType.Dataowner, + associatedUrn: urn, __typename: 'Owner', }, ], diff --git a/datahub-web-react/src/graphql/browse.graphql b/datahub-web-react/src/graphql/browse.graphql index 0b3a259e23cc65..ade935efa0dc92 100644 --- a/datahub-web-react/src/graphql/browse.graphql +++ b/datahub-web-react/src/graphql/browse.graphql @@ -152,6 +152,33 @@ query getBrowseResults($input: BrowseInput!) { ...entityDomain } } + ... on DataJob { + urn + type + dataFlow { + ...nonRecursiveDataFlowFields + } + jobId + ownership { + ...ownershipFields + } + properties { + name + description + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + editableProperties { + description + } + domain { + ...entityDomain + } + } ... on MLFeatureTable { urn type diff --git a/datahub-web-react/src/graphql/chart.graphql b/datahub-web-react/src/graphql/chart.graphql index 60e10c6aec4b90..020fe09830fa48 100644 --- a/datahub-web-react/src/graphql/chart.graphql +++ b/datahub-web-react/src/graphql/chart.graphql @@ -56,17 +56,14 @@ query getChart($urn: String!) { dashboards: relationships(input: { types: ["Contains"], direction: INCOMING, start: 0, count: 100 }) { ...fullRelationshipResults } - container { - ...entityContainer - } parentContainers { ...parentContainersFields } upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } status { removed @@ -74,6 +71,25 @@ query getChart($urn: String!) { dataPlatformInstance { ...dataPlatformInstanceFields } + statsSummary { + viewCount + uniqueUserCountLast30Days + topUsersLast30Days { + urn + type + username + properties { + displayName + firstName + lastName + fullName + } + editableProperties { + displayName + pictureLink + } + } + } } } diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql index a009d149bffdf9..6da6264f14df8c 100644 --- a/datahub-web-react/src/graphql/dashboard.graphql +++ b/datahub-web-react/src/graphql/dashboard.graphql @@ -1,17 +1,17 @@ query getDashboard($urn: String!) { dashboard(urn: $urn) { ...dashboardFields - editableProperties { - description - } charts: relationships(input: { types: ["Contains"], direction: OUTGOING, start: 0, count: 100 }) { ...fullRelationshipResults } + datasets: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) { + ...fullRelationshipResults + } upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } } } diff --git a/datahub-web-react/src/graphql/dataFlow.graphql b/datahub-web-react/src/graphql/dataFlow.graphql index dedf065d36a19c..72d3fd1b51fdee 100644 --- a/datahub-web-react/src/graphql/dataFlow.graphql +++ b/datahub-web-react/src/graphql/dataFlow.graphql @@ -50,10 +50,10 @@ query getDataFlow($urn: String!) { dataFlow(urn: $urn) { ...dataFlowFields upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } childJobs: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 }) { start @@ -69,6 +69,9 @@ query getDataFlow($urn: String!) { urn type orchestrator + platform { + ...platformFields + } } ownership { ...ownershipFields diff --git a/datahub-web-react/src/graphql/dataJob.graphql b/datahub-web-react/src/graphql/dataJob.graphql index 0353d09a48a1ec..556b474ce51883 100644 --- a/datahub-web-react/src/graphql/dataJob.graphql +++ b/datahub-web-react/src/graphql/dataJob.graphql @@ -1,41 +1,17 @@ query getDataJob($urn: String!) { dataJob(urn: $urn) { ...dataJobFields - parentFlow: relationships(input: { types: ["IsPartOf"], direction: OUTGOING, start: 0, count: 1 }) { - ...fullRelationshipResults - } - incoming: relationships( - input: { - types: ["DownstreamOf", "Consumes", "Produces", "TrainedBy"] - direction: INCOMING - start: 0 - count: 100 - } - ) { - ...fullRelationshipResults - } - outgoing: relationships( - input: { types: ["DownstreamOf", "Consumes", "Produces"], direction: OUTGOING, start: 0, count: 100 } - ) { - ...fullRelationshipResults - } upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults - } - status { - removed + ...partialLineageResults } runs(start: 0, count: 20) { count start total } - deprecation { - ...deprecationFields - } } } diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index e22497b457fa21..b964a223a273f4 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -24,148 +24,185 @@ query getDataProfiles($urn: String!, $limit: Int, $startTime: Long, $endTime: Lo query getDataset($urn: String!) { dataset(urn: $urn) { - ...nonRecursiveDatasetFields - schemaMetadata(version: 0) { - ...schemaMetadataFields - } - previousSchemaMetadata: schemaMetadata(version: -1) { - ...schemaMetadataFields - } - editableSchemaMetadata { - editableSchemaFieldInfo { - fieldPath - description - globalTags { - ...globalTagsFields - } - glossaryTerms { - ...glossaryTerms - } + ...nonSiblingDatasetFields + siblings { + isPrimary + siblings { + urn + type + ...nonSiblingDatasetFields } } - deprecation { - actor - deprecated - note - decommissionTime - } - globalTags { - ...globalTagsFields - } - glossaryTerms { - ...glossaryTerms - } - subTypes { - typeNames - } - domain { - ...entityDomain - } - container { - ...entityContainer - } - parentContainers { - ...parentContainersFields - } - usageStats(range: MONTH) { - buckets { - bucket - duration - resource - metrics { - uniqueUserCount - totalSqlQueries - topSqlQueries - } + } +} + +fragment nonSiblingDatasetFields on Dataset { + ...nonRecursiveDatasetFields + schemaMetadata(version: 0) { + ...schemaMetadataFields + } + editableSchemaMetadata { + editableSchemaFieldInfo { + fieldPath + description + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms } - aggregations { + } + } + deprecation { + actor + deprecated + note + decommissionTime + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + subTypes { + typeNames + } + domain { + ...entityDomain + } + parentContainers { + ...parentContainersFields + } + usageStats(range: MONTH) { + buckets { + bucket + duration + resource + metrics { uniqueUserCount totalSqlQueries - users { - user { - urn - username + topSqlQueries + } + } + aggregations { + uniqueUserCount + totalSqlQueries + users { + user { + urn + type + username + properties { + displayName + firstName + lastName + fullName + } + editableProperties { + displayName + pictureLink } - count - userEmail - } - fields { - fieldName - count } + count + userEmail } - } - datasetProfiles(limit: 1) { - rowCount - columnCount - timestampMillis - fieldProfiles { - fieldPath - uniqueCount - uniqueProportion - nullCount - nullProportion - min - max - mean - median - stdev - sampleValues + fields { + fieldName + count } } - health { - status - message - causes - } - assertions(start: 0, count: 1) { - total - } - operations(limit: 1) { - timestampMillis - lastUpdatedTimestamp - } - upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + } + datasetProfiles(limit: 1) { + rowCount + columnCount + timestampMillis + fieldProfiles { + fieldPath + uniqueCount + uniqueProportion + nullCount + nullProportion + min + max + mean + median + stdev + sampleValues } - downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + } + health { + type + status + message + causes + } + assertions(start: 0, count: 1) { + total + } + operations(limit: 1) { + timestampMillis + lastUpdatedTimestamp + } + upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { + ...partialLineageResults + } + downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { + ...partialLineageResults + } + ...viewProperties + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + aspectName + payload + renderSpec { + displayType + displayName + key } - ...viewProperties - autoRenderAspects: aspects(input: { autoRenderOnly: true }) { - aspectName - payload - renderSpec { - displayType - displayName - key + } + status { + removed + } + readRuns: runs(start: 0, count: 20, direction: INCOMING) { + count + start + total + } + writeRuns: runs(start: 0, count: 20, direction: OUTGOING) { + count + start + total + } + testResults { + passing { + test { + ...testFields } + type } - status { - removed - } - readRuns: runs(start: 0, count: 20, direction: INCOMING) { - count - start - total - } - writeRuns: runs(start: 0, count: 20, direction: OUTGOING) { - count - start - total + failing { + test { + ...testFields + } + type } - testResults { - passing { - test { - ...testFields - } - type + } + statsSummary { + queryCountLast30Days + uniqueUserCountLast30Days + topUsersLast30Days { + urn + type + username + properties { + displayName + firstName + lastName + fullName } - failing { - test { - ...testFields - } - type + editableProperties { + displayName + pictureLink } } } @@ -185,22 +222,34 @@ fragment viewProperties on Dataset { } } +fragment assertionsQuery on Dataset { + assertions(start: 0, count: 100) { + start + count + total + assertions { + ...assertionDetails + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + ...assertionRunEventDetails + } + } + } + } +} + query getDatasetAssertions($urn: String!) { dataset(urn: $urn) { - assertions(start: 0, count: 100) { - start - count - total - assertions { - ...assertionDetails - runEvents(status: COMPLETE, limit: 1) { - total - failed - succeeded - runEvents { - ...assertionRunEventDetails - } - } + ...assertionsQuery + siblings { + isPrimary + siblings { + urn + type + ...assertionsQuery } } } diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 098448993925d5..13c86ea1c1d4cf 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -53,4 +53,8 @@ query listDomains($input: ListDomainsInput!) { mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) -} \ No newline at end of file +} + +mutation deleteDomain($urn: String!) { + deleteDomain(urn: $urn) +} diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index 7921abd79a004d..8ac5193d12ab02 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -8,6 +8,7 @@ fragment globalTagsFields on GlobalTags { colorHex } } + associatedUrn } } @@ -17,16 +18,9 @@ fragment glossaryNode on GlossaryNode { properties { name } - children: relationships( - input: { - types: ["IsPartOf"] - direction: INCOMING - start: 0 - count: 1000 - } - ) { - count - } + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 1000 }) { + count + } } fragment glossaryTerm on GlossaryTerm { @@ -46,6 +40,7 @@ fragment glossaryTerms on GlossaryTerms { term { ...glossaryTerm } + associatedUrn } } @@ -154,6 +149,7 @@ fragment ownershipFields on Ownership { } } type + associatedUrn } lastModified { time @@ -209,9 +205,6 @@ fragment nonRecursiveDatasetFields on Dataset { schemaMetadata(version: 0) { ...schemaMetadataFields } - previousSchemaMetadata: schemaMetadata(version: -1) { - ...schemaMetadataFields - } editableSchemaMetadata { editableSchemaFieldInfo { fieldPath @@ -306,17 +299,6 @@ fragment dataJobFields on DataJob { ownership { ...ownershipFields } - inputOutput { - inputDatasets { - ...nonRecursiveDatasetFields - } - outputDatasets { - ...nonRecursiveDatasetFields - } - inputDatajobs { - ...nonRecursiveDataJobFields - } - } properties { name description @@ -344,6 +326,9 @@ fragment dataJobFields on DataJob { deprecation { ...deprecationFields } + status { + removed + } dataPlatformInstance { ...dataPlatformInstanceFields } @@ -371,6 +356,9 @@ fragment dashboardFields on Dashboard { time } } + editableProperties { + description + } ownership { ...ownershipFields } @@ -389,9 +377,6 @@ fragment dashboardFields on Dashboard { domain { ...entityDomain } - container { - ...entityContainer - } parentContainers { ...parentContainersFields } @@ -404,6 +389,25 @@ fragment dashboardFields on Dashboard { dataPlatformInstance { ...dataPlatformInstanceFields } + statsSummary { + viewCount + uniqueUserCountLast30Days + topUsersLast30Days { + urn + type + username + properties { + displayName + firstName + lastName + fullName + } + editableProperties { + displayName + pictureLink + } + } + } } fragment nonRecursiveMLFeature on MLFeature { @@ -798,7 +802,7 @@ fragment nonConflictingPlatformFields on DataPlatform { } } -fragment dataPlatformInstanceFields on DataPlatformInstance { +fragment dataPlatformInstanceFields on DataPlatformInstance { urn type platform { @@ -823,9 +827,12 @@ fragment entityContainer on Container { } } -fragment entityDomain on Domain { - urn - properties { - name +fragment entityDomain on DomainAssociation { + domain { + urn + properties { + name + } } + associatedUrn } diff --git a/datahub-web-react/src/graphql/glossaryTerm.graphql b/datahub-web-react/src/graphql/glossaryTerm.graphql index 55e82e85fe1419..1a6da8b0e00924 100644 --- a/datahub-web-react/src/graphql/glossaryTerm.graphql +++ b/datahub-web-react/src/graphql/glossaryTerm.graphql @@ -78,3 +78,11 @@ mutation createGlossaryTerm($input: CreateGlossaryEntityInput!) { mutation createGlossaryNode($input: CreateGlossaryEntityInput!) { createGlossaryNode(input: $input) } + +mutation addRelatedTerms($input: RelatedTermsInput!) { + addRelatedTerms(input: $input) +} + +mutation removeRelatedTerms($input: RelatedTermsInput!) { + removeRelatedTerms(input: $input) +} diff --git a/datahub-web-react/src/graphql/group.graphql b/datahub-web-react/src/graphql/group.graphql index 39dd6e891c4d9a..a992187f2a0c89 100644 --- a/datahub-web-react/src/graphql/group.graphql +++ b/datahub-web-react/src/graphql/group.graphql @@ -3,6 +3,10 @@ query getGroup($urn: String!, $membersCount: Int!) { urn type name + origin { + type + externalType + } info { displayName description @@ -21,7 +25,14 @@ query getGroup($urn: String!, $membersCount: Int!) { ownership { ...ownershipFields } - relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: 0, count: $membersCount }) { + relationships( + input: { + types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"] + direction: INCOMING + start: 0 + count: $membersCount + } + ) { start count total @@ -50,6 +61,47 @@ query getGroup($urn: String!, $membersCount: Int!) { } } +query getAllGroupMembers($urn: String!, $start: Int!, $count: Int!) { + corpGroup(urn: $urn) { + relationships( + input: { + types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"] + direction: INCOMING + start: $start + count: $count + } + ) { + start + count + total + relationships { + entity { + ... on CorpUser { + urn + username + info { + active + displayName + title + firstName + lastName + fullName + } + properties { + displayName + } + editableProperties { + displayName + title + pictureLink + } + } + } + } + } + } +} + query getGroupMembers($urn: String!, $start: Int!, $count: Int!) { corpGroup(urn: $urn) { relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: $start, count: $count }) { @@ -84,6 +136,40 @@ query getGroupMembers($urn: String!, $start: Int!, $count: Int!) { } } +query getNativeGroupMembers($urn: String!, $start: Int!, $count: Int!) { + corpGroup(urn: $urn) { + relationships(input: { types: ["IsMemberOfNativeGroup"], direction: INCOMING, start: $start, count: $count }) { + start + count + total + relationships { + entity { + ... on CorpUser { + urn + username + info { + active + displayName + title + firstName + lastName + fullName + } + properties { + displayName + } + editableProperties { + displayName + title + pictureLink + } + } + } + } + } + } +} + query listGroups($input: ListGroupsInput!) { listGroups(input: $input) { start @@ -93,12 +179,18 @@ query listGroups($input: ListGroupsInput!) { urn type name + origin { + type + externalType + } info { displayName description email } - memberCount: relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: 0, count: 1 }) { + memberCount: relationships( + input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"], direction: INCOMING, start: 0, count: 1 } + ) { total } } @@ -125,4 +217,4 @@ mutation updateCorpGroupProperties($urn: String!, $input: CorpGroupUpdateInput!) updateCorpGroupProperties(urn: $urn, input: $input) { urn } -} +} \ No newline at end of file diff --git a/datahub-web-react/src/graphql/ingestion.graphql b/datahub-web-react/src/graphql/ingestion.graphql index b4f503f0783edc..f8529bc356c406 100644 --- a/datahub-web-react/src/graphql/ingestion.graphql +++ b/datahub-web-react/src/graphql/ingestion.graphql @@ -22,6 +22,7 @@ query listIngestionSources($input: ListIngestionSourcesInput!) { total executionRequests { urn + id input { requestedAt } @@ -56,6 +57,7 @@ query getIngestionSource($urn: String!, $runStart: Int, $runCount: Int) { total executionRequests { urn + id input { requestedAt source { @@ -75,6 +77,7 @@ query getIngestionSource($urn: String!, $runStart: Int, $runCount: Int) { query getIngestionExecutionRequest($urn: String!) { executionRequest(urn: $urn) { urn + id input { source { type @@ -85,6 +88,10 @@ query getIngestionExecutionRequest($urn: String!) { startTimeMs durationMs report + structuredReport { + type + serializedValue + } } } } @@ -129,3 +136,7 @@ mutation createIngestionExecutionRequest($input: CreateIngestionExecutionRequest mutation cancelIngestionExecutionRequest($input: CancelIngestionExecutionRequestInput!) { cancelIngestionExecutionRequest(input: $input) } + +mutation createTestConnectionRequest($input: CreateTestConnectionRequestInput!) { + createTestConnectionRequest(input: $input) +} diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index e390644c968480..5f898a12ebb173 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -1,8 +1,40 @@ -fragment relationshipFields on EntityWithRelationships { +fragment lineageNodeProperties on EntityWithRelationships { urn type ... on DataJob { - ...dataJobFields + urn + type + dataFlow { + ...nonRecursiveDataFlowFields + } + jobId + ownership { + ...ownershipFields + } + properties { + name + description + externalUrl + customProperties { + key + value + } + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + domain { + ...entityDomain + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } editableProperties { description } @@ -42,13 +74,52 @@ fragment relationshipFields on EntityWithRelationships { } } ... on Dashboard { - ...dashboardFields - editableProperties { + urn + type + tool + dashboardId + properties { + name description + externalUrl + lastRefreshed + created { + time + } + lastModified { + time + } + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms } platform { ...platformFields } + domain { + ...entityDomain + } + parentContainers { + ...parentContainersFields + } + status { + removed + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + editableProperties { + description + } status { removed } @@ -81,6 +152,7 @@ fragment relationshipFields on EntityWithRelationships { properties { name description + qualifiedName } editableProperties { description @@ -139,10 +211,24 @@ fragment relationshipFields on EntityWithRelationships { ... on MLPrimaryKey { ...nonRecursiveMLPrimaryKey } - upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { +} + +fragment lineageFields on EntityWithRelationships { + ...lineageNodeProperties + ... on Dataset { + siblings { + isPrimary + siblings { + urn + type + ...lineageNodeProperties + } + } + } + upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100, separateSiblings: $separateSiblings }) { ...leafLineageResults } - downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { + downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100, separateSiblings: $separateSiblings }) { ...leafLineageResults } } @@ -154,7 +240,7 @@ fragment fullLineageResults on EntityLineageResult { relationships { type entity { - ...relationshipFields + ...lineageFields } } } @@ -171,3 +257,39 @@ fragment leafLineageResults on EntityLineageResult { } } } + +fragment partialLineageResults on EntityLineageResult { + start + count + total +} + +query getEntityLineage($urn: String!, $separateSiblings: Boolean) { + entity(urn: $urn) { + urn + type + ...lineageNodeProperties + ... on Dataset { + siblings { + isPrimary + siblings { + urn + type + ...lineageNodeProperties + } + } + } + ... on EntityWithRelationships { + upstream: lineage( + input: { direction: UPSTREAM, start: 0, count: 100, separateSiblings: $separateSiblings } + ) { + ...fullLineageResults + } + downstream: lineage( + input: { direction: DOWNSTREAM, start: 0, count: 100, separateSiblings: $separateSiblings } + ) { + ...fullLineageResults + } + } + } +} diff --git a/datahub-web-react/src/graphql/me.graphql b/datahub-web-react/src/graphql/me.graphql index 3e9bb112026b08..bf64cedd59084f 100644 --- a/datahub-web-react/src/graphql/me.graphql +++ b/datahub-web-react/src/graphql/me.graphql @@ -30,6 +30,10 @@ query getMe { manageDomains manageTests manageGlossaries + manageUserCredentials + manageTags + createDomains + createTags } } } diff --git a/datahub-web-react/src/graphql/mlFeature.graphql b/datahub-web-react/src/graphql/mlFeature.graphql index d1428c643d32c0..120a6453b76f42 100644 --- a/datahub-web-react/src/graphql/mlFeature.graphql +++ b/datahub-web-react/src/graphql/mlFeature.graphql @@ -2,10 +2,10 @@ query getMLFeature($urn: String!) { mlFeature(urn: $urn) { ...nonRecursiveMLFeature upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } featureTables: relationships(input: { types: ["Contains"], direction: INCOMING, start: 0, count: 100 }) { ...fullRelationshipResults diff --git a/datahub-web-react/src/graphql/mlFeatureTable.graphql b/datahub-web-react/src/graphql/mlFeatureTable.graphql index de8b5206bc889c..ebb19b89598c66 100644 --- a/datahub-web-react/src/graphql/mlFeatureTable.graphql +++ b/datahub-web-react/src/graphql/mlFeatureTable.graphql @@ -2,10 +2,10 @@ query getMLFeatureTable($urn: String!) { mlFeatureTable(urn: $urn) { ...nonRecursiveMLFeatureTable upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } } } diff --git a/datahub-web-react/src/graphql/mlModel.graphql b/datahub-web-react/src/graphql/mlModel.graphql index 08d9ef149acce4..5d60be86a31c2b 100644 --- a/datahub-web-react/src/graphql/mlModel.graphql +++ b/datahub-web-react/src/graphql/mlModel.graphql @@ -2,10 +2,10 @@ query getMLModel($urn: String!) { mlModel(urn: $urn) { ...nonRecursiveMLModel upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } features: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) { ...fullRelationshipResults diff --git a/datahub-web-react/src/graphql/mlModelGroup.graphql b/datahub-web-react/src/graphql/mlModelGroup.graphql index 65caa6ab96b4d0..635fc8cd75b910 100644 --- a/datahub-web-react/src/graphql/mlModelGroup.graphql +++ b/datahub-web-react/src/graphql/mlModelGroup.graphql @@ -22,10 +22,10 @@ query getMLModelGroup($urn: String!) { ...fullRelationshipResults } upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } } } diff --git a/datahub-web-react/src/graphql/mlPrimaryKey.graphql b/datahub-web-react/src/graphql/mlPrimaryKey.graphql index 576e9f8031326d..004fba61a2f96c 100644 --- a/datahub-web-react/src/graphql/mlPrimaryKey.graphql +++ b/datahub-web-react/src/graphql/mlPrimaryKey.graphql @@ -2,10 +2,10 @@ query getMLPrimaryKey($urn: String!) { mlPrimaryKey(urn: $urn) { ...nonRecursiveMLPrimaryKey upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { - ...fullLineageResults + ...partialLineageResults } featureTables: relationships(input: { types: ["KeyedBy"], direction: INCOMING, start: 0, count: 100 }) { ...fullRelationshipResults diff --git a/datahub-web-react/src/graphql/mutations.graphql b/datahub-web-react/src/graphql/mutations.graphql index 71746971443ab9..beeaf96acfefd5 100644 --- a/datahub-web-react/src/graphql/mutations.graphql +++ b/datahub-web-react/src/graphql/mutations.graphql @@ -2,18 +2,34 @@ mutation removeTag($input: TagAssociationInput!) { removeTag(input: $input) } +mutation batchRemoveTags($input: BatchRemoveTagsInput!) { + batchRemoveTags(input: $input) +} + mutation addTag($input: TagAssociationInput!) { addTag(input: $input) } +mutation batchAddTags($input: BatchAddTagsInput!) { + batchAddTags(input: $input) +} + mutation removeTerm($input: TermAssociationInput!) { removeTerm(input: $input) } +mutation batchRemoveTerms($input: BatchRemoveTermsInput!) { + batchRemoveTerms(input: $input) +} + mutation addTerm($input: TermAssociationInput!) { addTerm(input: $input) } +mutation batchAddTerms($input: BatchAddTermsInput!) { + batchAddTerms(input: $input) +} + mutation addLink($input: AddLinkInput!) { addLink(input: $input) } @@ -26,10 +42,18 @@ mutation addOwner($input: AddOwnerInput!) { addOwner(input: $input) } +mutation batchAddOwners($input: BatchAddOwnersInput!) { + batchAddOwners(input: $input) +} + mutation removeOwner($input: RemoveOwnerInput!) { removeOwner(input: $input) } +mutation batchRemoveOwners($input: BatchRemoveOwnersInput!) { + batchRemoveOwners(input: $input) +} + mutation updateDescription($input: DescriptionUpdateInput!) { updateDescription(input: $input) } @@ -65,3 +89,15 @@ mutation addTerms($input: AddTermsInput!) { mutation updateName($input: UpdateNameInput!) { updateName(input: $input) } + +mutation batchSetDomain($input: BatchSetDomainInput!) { + batchSetDomain(input: $input) +} + +mutation batchUpdateDeprecation($input: BatchUpdateDeprecationInput!) { + batchUpdateDeprecation(input: $input) +} + +mutation batchUpdateSoftDeleted($input: BatchUpdateSoftDeletedInput!) { + batchUpdateSoftDeleted(input: $input) +} diff --git a/datahub-web-react/src/graphql/preview.graphql b/datahub-web-react/src/graphql/preview.graphql index 2980f7c1ff1c2b..6d1b1b8de3b636 100644 --- a/datahub-web-react/src/graphql/preview.graphql +++ b/datahub-web-react/src/graphql/preview.graphql @@ -61,7 +61,9 @@ fragment entityPreview on Entity { displayName description } - memberCount: relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: 0, count: 1 }) { + memberCount: relationships( + input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"], direction: INCOMING, start: 0, count: 1 } + ) { total } } @@ -220,6 +222,12 @@ fragment entityPreview on Entity { ...deprecationFields } } + ... on GlossaryNode { + properties { + name + description + } + } ... on MLFeatureTable { urn type diff --git a/datahub-web-react/src/graphql/relationships.graphql b/datahub-web-react/src/graphql/relationships.graphql index 54bc0fd92c2d12..e2afff5dfa6d27 100644 --- a/datahub-web-react/src/graphql/relationships.graphql +++ b/datahub-web-react/src/graphql/relationships.graphql @@ -23,3 +23,23 @@ fragment leafRelationshipResults on EntityRelationshipsResult { } } } + +fragment relationshipFields on EntityWithRelationships { + ...lineageNodeProperties + ... on Dataset { + siblings { + isPrimary + siblings { + urn + type + ...lineageNodeProperties + } + } + } + upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { + ...leafLineageResults + } + downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { + ...leafLineageResults + } +} diff --git a/datahub-web-react/src/graphql/schemaBlame.graphql b/datahub-web-react/src/graphql/schemaBlame.graphql index f5ec131fec36d3..4fafaa4340c887 100644 --- a/datahub-web-react/src/graphql/schemaBlame.graphql +++ b/datahub-web-react/src/graphql/schemaBlame.graphql @@ -17,8 +17,8 @@ query getSchemaBlame($input: GetSchemaBlameInput!) { } } -query getSchemaBlameVersions($input: GetSchemaBlameInput!) { - getSchemaBlame(input: $input) { +query getSchemaVersionList($input: GetSchemaVersionListInput!) { + getSchemaVersionList(input: $input) { latestVersion { semanticVersion semanticVersionTimestamp diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 25926b97b46a70..f33fc5c386d565 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -1,9 +1,175 @@ +fragment autoCompleteFields on Entity { + urn + type + ... on Dataset { + name + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + properties { + name + qualifiedName + } + } + ... on CorpUser { + username + properties { + displayName + title + firstName + lastName + fullName + } + editableProperties { + displayName + } + } + ... on CorpGroup { + name + info { + displayName + } + } + ... on Dashboard { + properties { + name + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on Chart { + chartId + properties { + name + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on DataFlow { + orchestrator + properties { + name + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on DataJob { + dataFlow { + orchestrator + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + jobId + properties { + name + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on GlossaryTerm { + name + hierarchicalName + properties { + name + } + } + ... on GlossaryNode { + properties { + name + } + } + ... on Domain { + properties { + name + } + } + ... on Container { + properties { + name + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + parentContainers { + ...parentContainersFields + } + } + ... on Tag { + name + properties { + name + } + } + ... on MLFeatureTable { + name + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on MLFeature { + name + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on MLPrimaryKey { + name + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on MLModel { + name + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on MLModelGroup { + name + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } +} + query getAutoCompleteResults($input: AutoCompleteInput!) { autoComplete(input: $input) { query suggestions entities { - ...searchResultFields + ...autoCompleteFields } } } @@ -15,7 +181,7 @@ query getAutoCompleteMultipleResults($input: AutoCompleteMultipleInput!) { type suggestions entities { - ...searchResultFields + ...autoCompleteFields } } } @@ -62,15 +228,57 @@ fragment searchResultFields on Entity { domain { ...entityDomain } - container { - ...entityContainer - } parentContainers { ...parentContainersFields } deprecation { ...deprecationFields } + siblings { + isPrimary + siblings { + urn + type + ... on Dataset { + platform { + ...platformFields + } + name + properties { + name + description + qualifiedName + } + } + } + } + lastProfile: datasetProfiles(limit: 1) { + rowCount + timestampMillis + } + lastOperation: operations(limit: 1) { + lastUpdatedTimestamp + timestampMillis + } + statsSummary { + queryCountLast30Days + uniqueUserCountLast30Days + topUsersLast30Days { + urn + type + username + properties { + displayName + firstName + lastName + fullName + } + editableProperties { + displayName + pictureLink + } + } + } } ... on CorpUser { username @@ -104,14 +312,13 @@ fragment searchResultFields on Entity { displayName description } - memberCount: relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: 0, count: 1 }) { + memberCount: relationships( + input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"], direction: INCOMING, start: 0, count: 1 } + ) { total } } ... on Dashboard { - urn - type - tool dashboardId properties { name @@ -143,20 +350,33 @@ fragment searchResultFields on Entity { domain { ...entityDomain } - container { - ...entityContainer - } deprecation { ...deprecationFields } parentContainers { ...parentContainersFields } + statsSummary { + viewCount + uniqueUserCountLast30Days + topUsersLast30Days { + urn + type + username + properties { + displayName + firstName + lastName + fullName + } + editableProperties { + displayName + pictureLink + } + } + } } ... on Chart { - urn - type - tool chartId properties { name @@ -167,6 +387,9 @@ fragment searchResultFields on Entity { lastModified { time } + created { + time + } } ownership { ...ownershipFields @@ -189,20 +412,33 @@ fragment searchResultFields on Entity { domain { ...entityDomain } - container { - ...entityContainer - } deprecation { ...deprecationFields } parentContainers { ...parentContainersFields } + statsSummary { + viewCount + uniqueUserCountLast30Days + topUsersLast30Days { + urn + type + username + properties { + displayName + firstName + lastName + fullName + } + editableProperties { + displayName + pictureLink + } + } + } } ... on DataFlow { - urn - type - orchestrator flowId cluster properties { @@ -234,10 +470,11 @@ fragment searchResultFields on Entity { deprecation { ...deprecationFields } + childJobs: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 }) { + total + } } ... on DataJob { - urn - type dataFlow { ...nonRecursiveDataFlowFields } @@ -267,6 +504,19 @@ fragment searchResultFields on Entity { dataPlatformInstance { ...dataPlatformInstanceFields } + lastRun: runs(start: 0, count: 1) { + count + start + total + runs { + urn + type + created { + time + actor + } + } + } } ... on GlossaryTerm { name @@ -292,10 +542,8 @@ fragment searchResultFields on Entity { parentNodes { ...parentNodesFields } - } ... on Domain { - urn properties { name description @@ -305,7 +553,6 @@ fragment searchResultFields on Entity { } } ... on Container { - urn properties { name description @@ -325,9 +572,6 @@ fragment searchResultFields on Entity { tags { ...globalTagsFields } - institutionalMemory { - ...institutionalMemoryFields - } glossaryTerms { ...glossaryTerms } @@ -337,9 +581,6 @@ fragment searchResultFields on Entity { entities(input: {}) { total } - container { - ...entityContainer - } deprecation { ...deprecationFields } @@ -348,8 +589,6 @@ fragment searchResultFields on Entity { } } ... on MLFeatureTable { - urn - type name description featureTableProperties { @@ -416,6 +655,9 @@ fragment searchResultFields on Entity { } ... on Tag { name + properties { + name + } description } } @@ -431,12 +673,12 @@ fragment facetFields on FacetMetadata { type ... on Tag { name - description properties { colorHex } } ... on GlossaryTerm { + name properties { name } @@ -448,13 +690,11 @@ fragment facetFields on FacetMetadata { ...dataPlatformInstanceFields } ... on Domain { - urn properties { name } } ... on Container { - urn platform { ...platformFields } @@ -463,7 +703,6 @@ fragment facetFields on FacetMetadata { } } ... on CorpUser { - urn username properties { displayName @@ -475,7 +714,6 @@ fragment facetFields on FacetMetadata { } } ... on CorpGroup { - urn name properties { displayName diff --git a/datahub-web-react/src/graphql/tag.graphql b/datahub-web-react/src/graphql/tag.graphql index e2404123d1205b..37aaebc2650322 100644 --- a/datahub-web-react/src/graphql/tag.graphql +++ b/datahub-web-react/src/graphql/tag.graphql @@ -4,9 +4,9 @@ query getTag($urn: String!) { name description properties { - name - description - colorHex + name + description + colorHex } ownership { ...ownershipFields @@ -23,4 +23,12 @@ mutation updateTag($urn: String!, $input: TagUpdateInput!) { ...ownershipFields } } -} \ No newline at end of file +} + +mutation deleteTag($urn: String!) { + deleteTag(urn: $urn) +} + +mutation createTag($input: CreateTagInput!) { + createTag(input: $input) +} diff --git a/datahub-web-react/src/graphql/user.graphql b/datahub-web-react/src/graphql/user.graphql index 9cc318152dc2b2..8472beed5ac7fb 100644 --- a/datahub-web-react/src/graphql/user.graphql +++ b/datahub-web-react/src/graphql/user.graphql @@ -2,6 +2,7 @@ query getUser($urn: String!, $groupsCount: Int!) { corpUser(urn: $urn) { urn username + isNativeUser info { active displayName @@ -26,7 +27,14 @@ query getUser($urn: String!, $groupsCount: Int!) { globalTags { ...globalTagsFields } - relationships(input: { types: ["IsMemberOfGroup"], direction: OUTGOING, start: 0, count: $groupsCount }) { + relationships( + input: { + types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"] + direction: OUTGOING + start: 0 + count: $groupsCount + } + ) { start count total @@ -41,7 +49,9 @@ query getUser($urn: String!, $groupsCount: Int!) { description email } - relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING }) { + relationships( + input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"], direction: INCOMING } + ) { start count total @@ -55,7 +65,14 @@ query getUser($urn: String!, $groupsCount: Int!) { query getUserGroups($urn: String!, $start: Int!, $count: Int!) { corpUser(urn: $urn) { - relationships(input: { types: ["IsMemberOfGroup"], direction: OUTGOING, start: $start, count: $count }) { + relationships( + input: { + types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"] + direction: OUTGOING + start: $start + count: $count + } + ) { start count total @@ -71,7 +88,7 @@ query getUserGroups($urn: String!, $start: Int!, $count: Int!) { email } } - relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING }) { + relationships(input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"], direction: INCOMING }) { start count total @@ -90,6 +107,7 @@ query listUsers($input: ListUsersInput!) { users { urn username + isNativeUser info { active displayName @@ -124,3 +142,22 @@ mutation updateCorpUserProperties($urn: String!, $input: CorpUserUpdateInput!) { urn } } + +mutation createNativeUserInviteToken { + createNativeUserInviteToken { + inviteToken + } +} + +query getNativeUserInviteToken { + getNativeUserInviteToken { + inviteToken + } +} + +mutation createNativeUserResetToken($input: CreateNativeUserResetTokenInput!) { + createNativeUserResetToken(input: $input) { + resetToken + } +} + diff --git a/datahub-web-react/src/graphql/versionedDataset.graphql b/datahub-web-react/src/graphql/versionedDataset.graphql index b9f3002dc09997..d61139927e14b3 100644 --- a/datahub-web-react/src/graphql/versionedDataset.graphql +++ b/datahub-web-react/src/graphql/versionedDataset.graphql @@ -11,6 +11,7 @@ query getVersionedDataset($urn: String!, $versionStamp: String) { recursive isPartOfKey } + lastObserved } editableSchemaMetadata { editableSchemaFieldInfo { diff --git a/datahub-web-react/src/images/deltalakelogo.png b/datahub-web-react/src/images/deltalakelogo.png new file mode 100644 index 00000000000000..28f9a9d28a0cb6 Binary files /dev/null and b/datahub-web-react/src/images/deltalakelogo.png differ diff --git a/datahub-web-react/src/images/logo-salesforce.svg b/datahub-web-react/src/images/logo-salesforce.svg new file mode 100644 index 00000000000000..a9a4d2b04239c8 --- /dev/null +++ b/datahub-web-react/src/images/logo-salesforce.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index 0a4c49316e57fa..cd29f156016c73 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -2826,6 +2826,13 @@ resolved "https://registry.yarnpkg.com/@types/diff/-/diff-5.0.0.tgz#eb71e94feae62548282c4889308a3dfb57e36020" integrity sha512-jrm2K65CokCCX4NmowtA+MfXyuprZC13jbRuwprs6/04z/EcFg/MCwYdsHn+zgV4CQBiATiI7AEq7y1sZCtWKA== +"@types/dompurify@^2.3.3": + version "2.3.3" + resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.3.3.tgz#c24c92f698f77ed9cc9d9fa7888f90cf2bfaa23f" + integrity sha512-nnVQSgRVuZ/843oAfhA25eRSNzUFcBPk/LOiw5gm8mD9/X7CNcbRkQu/OsjCewO8+VIYfPxUnXvPEVGenw14+w== + dependencies: + "@types/trusted-types" "*" + "@types/eslint@^7.2.6": version "7.2.11" resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-7.2.11.tgz#180b58f5bb7d7376e39d22496e2b08901aa52fd2" @@ -3138,6 +3145,11 @@ dependencies: "@types/jest" "*" +"@types/trusted-types@*": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.2.tgz#fc25ad9943bcac11cceb8168db4f275e0e72e756" + integrity sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg== + "@types/uglify-js@*": version "3.13.0" resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.13.0.tgz#1cad8df1fb0b143c5aba08de5712ea9d1ff71124" @@ -5086,9 +5098,9 @@ buffer-equal-constant-time@1.0.1: integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + version "1.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== buffer-indexof@^1.0.0: version "1.1.1" @@ -6917,6 +6929,11 @@ domhandler@^2.3.0: dependencies: domelementtype "1" +dompurify@^2.3.8: + version "2.3.8" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f" + integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw== + domutils@^1.5.1, domutils@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" @@ -10763,7 +10780,7 @@ lodash.camelcase@^4.3.0: lodash.clonedeep@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" - integrity sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8= + integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== lodash.compact@^3.0.1: version "3.0.1" @@ -11589,6 +11606,11 @@ moment-timezone@^0.5.34: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== +monaco-editor@^0.28.1: + version "0.28.1" + resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.28.1.tgz#732788ff2172d59e6d436b206da8cac715413940" + integrity sha512-P1vPqxB4B1ZFzTeR1ScggSp9/5NoQrLCq88fnlNUsuRAP1usEBN4TIpI2lw0AYIZNVIanHk0qwjze2uJwGOHUw== + monaco-yaml@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/monaco-yaml/-/monaco-yaml-3.2.1.tgz#45ce9f7f8140dc26249ac99eb0a9e5a9ab9beb87" @@ -13982,6 +14004,11 @@ react-is@^17.0.0, react-is@^17.0.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== +react-js-cron@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/react-js-cron/-/react-js-cron-2.1.0.tgz#ce88a3260617222d8e1dc51534bb6606088304fc" + integrity sha512-mxpSS8WJAY6gRZ+XR8z22u4mRKfHmB4ej2BO3DKrNRZhTGdIdF19F45LZT6GKjQqVyWOYMK1mFafTvgqqoAXlQ== + react-markdown@6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-6.0.2.tgz#d89be45c278b1e5f0196f851fffb11e30c69f027" @@ -15310,9 +15337,9 @@ source-map-resolve@^0.6.0: decode-uri-component "^0.2.0" source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.12, source-map-support@~0.5.19: - version "0.5.19" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" - integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== dependencies: buffer-from "^1.0.0" source-map "^0.6.0" @@ -15992,9 +16019,9 @@ terser-webpack-plugin@^1.4.3: worker-farm "^1.7.0" terser@^4.1.2, terser@^4.6.2, terser@^4.6.3: - version "4.8.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.0.tgz#63056343d7c70bb29f3af665865a46fe03a0df17" - integrity sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw== + version "4.8.1" + resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.1.tgz#a00e5634562de2239fd404c649051bf6fc21144f" + integrity sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw== dependencies: commander "^2.20.0" source-map "~0.6.1" diff --git a/docker/README.md b/docker/README.md index 156c09bd59b6c8..0905bb22ef1c1c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,11 +23,19 @@ You can easily download and run all these images and their dependencies with our DataHub Docker Images: -* [linkedin/datahub-ingestion](https://hub.docker.com/r/linkedin/datahub-ingestion/tags?page=1&name=v) - This contains the Python CLI. If you are looking for docker image for every minor CLI release you can find them under [acryldata/datahub-ingestion](https://hub.docker.com/r/acryldata/datahub-ingestion/tags?page=1&name=v) -* [linkedin/datahub-gms](https://cloud.docker.com/repository/docker/linkedin/datahub-gms/) -* [linkedin/datahub-frontend-react](https://cloud.docker.com/repository/docker/linkedin/datahub-frontend-react/) -* [linkedin/datahub-mae-consumer](https://cloud.docker.com/repository/docker/linkedin/datahub-mae-consumer/) -* [linkedin/datahub-mce-consumer](https://cloud.docker.com/repository/docker/linkedin/datahub-mce-consumer/) +Do not use `latest` or `debug` tags for any of the image as those are not supported and present only due to leagcy reasons. Please use `head` or tags specific for versions like `v0.8.40`. For production we recommend using version specific tags not `head`. + +* [linkedin/datahub-ingestion](https://hub.docker.com/r/linkedin/datahub-ingestion/) - This contains the Python CLI. If you are looking for docker image for every minor CLI release you can find them under [acryldata/datahub-ingestion](https://hub.docker.com/r/acryldata/datahub-ingestion/). +* [linkedin/datahub-gms](https://hub.docker.com/repository/docker/linkedin/datahub-gms/). +* [linkedin/datahub-frontend-react](https://hub.docker.com/repository/docker/linkedin/datahub-frontend-react/) +* [linkedin/datahub-mae-consumer](https://hub.docker.com/repository/docker/linkedin/datahub-mae-consumer/) +* [linkedin/datahub-mce-consumer](https://hub.docker.com/repository/docker/linkedin/datahub-mce-consumer/) +* [acryldata/datahub-upgrade](https://hub.docker.com/r/acryldata/datahub-upgrade/) +* [linkedin/datahub-kafka-setup](https://hub.docker.com/r/acryldata/datahub-kafka-setup/) +* [linkedin/datahub-elasticsearch-setup](https://hub.docker.com/r/linkedin/datahub-elasticsearch-setup/) +* [acryldata/datahub-mysql-setup](https://hub.docker.com/r/acryldata/datahub-mysql-setup/) +* [acryldata/datahub-postgres-setup](https://hub.docker.com/r/acryldata/datahub-postgres-setup/) +* [acryldata/datahub-actions](https://hub.docker.com/r/acryldata/datahub-actions). Do not use `acryldata/acryl-datahub-actions` as that is deprecated and no longer used. Dependencies: * [Kafka, Zookeeper, and Schema Registry](kafka-setup) @@ -62,55 +70,3 @@ COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -p datahub build This is because we're relying on builtkit for multistage builds. It does not hurt also set `DATAHUB_VERSION` to something unique. -## Ember -To serve the legacy Ember UI, follow the instructions below. - -> **Before continuing**: If you've already run a deploy script, don't forget to clear containers using `docker container prune` - -### Serving Ember Only - -#### All Containers - -Use the `quickstart-ember.sh` script to launch all containers in DataHub, including a frontend server that serves the Ember UI -``` -./quickstart-ember.sh -``` - -#### The Bare Minimum -Run the following command to launch only the Ember server and its required dependencies - -``` -docker-compose -f docker-compose.ember.yml -f docker-compose.yml -f docker-compose.override.yml up datahub-frontend-ember -``` - -Once complete, navigate to `localhost:9001` in your browser to see the legacy Ember app. - -### Serving React + Ember -If you'd like to serve the React and Ember UIs side-by-side, you can deploy the `datahub-frontend-ember` service manually. - -#### All Containers - -To deploy all DataHub containers, run the quickstart script: -``` -./quickstart.sh -``` - -Next, deploy the container that serves the Ember UI: - -``` -docker-compose -f docker-compose.ember.yml -f docker-compose.yml -f docker-compose.override.yml up --no-deps datahub-frontend-ember -``` - -#### The Bare Minimum -First, start the React frontend server & its required dependencies: - -``` -docker-compose up datahub-frontend-react -``` - -Then, start the Ember frontend server & its required dependencies: -``` -docker-compose -f docker-compose.ember.yml -f docker-compose.yml -f docker-compose.override.yml up datahub-frontend-ember -``` - -Navigate to `localhost:9002/` to view the React app & `localhost:9001/` to view the legacy Ember app. diff --git a/docker/datahub-actions/env/docker.env b/docker/datahub-actions/env/docker.env index 7cc401745ed830..f055455c37d852 100644 --- a/docker/datahub-actions/env/docker.env +++ b/docker/datahub-actions/env/docker.env @@ -1,5 +1,6 @@ -GMS_HOST=datahub-gms -GMS_PORT=8080 +DATAHUB_GMS_HOST=datahub-gms +DATAHUB_GMS_PORT=8080 + KAFKA_BOOTSTRAP_SERVER=broker:29092 SCHEMA_REGISTRY_URL=http://schema-registry:8081 METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile index 24c8c82f002797..379c815e99159b 100644 --- a/docker/datahub-gms/Dockerfile +++ b/docker/datahub-gms/Dockerfile @@ -47,6 +47,8 @@ FROM base as dev-install FROM ${APP_ENV}-install as final +RUN mkdir -p /etc/datahub/plugins/auth/resources + RUN addgroup -S datahub && adduser -S datahub -G datahub USER datahub diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env index f68896bbb65f5d..6356b33d5eb32d 100644 --- a/docker/datahub-gms/env/docker-without-neo4j.env +++ b/docker/datahub-gms/env/docker-without-neo4j.env @@ -16,7 +16,7 @@ MAE_CONSUMER_ENABLED=true MCE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true -UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35 +UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 # Uncomment to disable persistence of client-side analytics events # DATAHUB_ANALYTICS_ENABLED=false diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index 7f8973ab38b04f..f74ece24907178 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -19,7 +19,7 @@ MAE_CONSUMER_ENABLED=true MCE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true -UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35 +UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 # Uncomment to enable Metadata Service Authentication #METADATA_SERVICE_AUTH_ENABLED=true @@ -32,6 +32,13 @@ UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35 # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment +# METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 +# FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME=FailedMetadataChangeProposal_v1 +# METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 +# METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME=MetadataChangeLog_Timeseries_v1 +# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1 +# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 +# Deprecated! # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 # METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4 @@ -57,7 +64,7 @@ UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35 # ELASTICSEARCH_PASSWORD= # To disable Analytics on the UI -# ANALYTICS_ENABLED=false +# DATAHUB_ANALYTICS_ENABLED=false # Encryption of DataHub Secrets # SECRET_SERVICE_ENCRYPTION_KEY= diff --git a/docker/datahub-gms/start.sh b/docker/datahub-gms/start.sh index be1257478312ea..cc4b42c4edfaf6 100755 --- a/docker/datahub-gms/start.sh +++ b/docker/datahub-gms/start.sh @@ -53,6 +53,10 @@ if [[ $ENABLE_PROMETHEUS == true ]]; then PROMETHEUS_AGENT="-javaagent:jmx_prometheus_javaagent.jar=4318:/datahub/datahub-gms/scripts/prometheus-config.yaml " fi +# for container based deployments the default directory is /etc/datahub/plugins/auth/resources and it can be different for +# kubernetes deployments +auth_resource_dir=${AUTH_RESOURCES_DIR:-"/etc/datahub/plugins/auth/resources"} +# Option --classes ${AUTH_RESOURCE_LOOK_UP_DIR} is added for Apache Ranger library to load the ranger-datahub-security.xml from classpath COMMON=" $WAIT_FOR_EBEAN \ $WAIT_FOR_CASSANDRA \ @@ -64,7 +68,7 @@ COMMON=" $PROMETHEUS_AGENT \ -jar /jetty-runner.jar \ --jar jetty-util.jar \ - --jar jetty-jmx.jar \ + --jar jetty-jmx.jar --classes ${auth_resource_dir} \ --config /datahub/datahub-gms/scripts/jetty.xml \ /datahub/datahub-gms/bin/war.war" diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 3cae6ecf16570e..aaea8b04f38362 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -1,20 +1,11 @@ # Defining environment ARG APP_ENV=prod -FROM python:3.8 as base +FROM acryldata/datahub-ingestion-base as base # ENV DOCKERIZE_VERSION v0.6.1 # RUN apk --no-cache add curl tar \ # && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.20.v20190813/jetty-runner-9.4.20.v20190813.jar --output jetty-runner.jar \ # && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv -RUN apt-get update && apt-get install -y \ - jq \ - librdkafka-dev \ - python3-ldap \ - libldap2-dev \ - libsasl2-dev \ - libsasl2-modules \ - ldap-utils \ - && python -m pip install --upgrade pip wheel setuptools==57.5.0 FROM openjdk:8 as prod-build @@ -29,11 +20,13 @@ RUN cd /datahub-src/metadata-ingestion && \ FROM base as prod-install COPY --from=prod-codegen /datahub-src/metadata-ingestion /datahub-ingestion +COPY --from=prod-codegen /root/.cache/pip /root/.cache/pip ARG RELEASE_VERSION RUN cd /datahub-ingestion && \ sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ cat src/datahub/__init__.py && \ - pip install ".[all]" + pip install ".[all]" && \ + pip freeze FROM base as dev-install # Dummy stage for development. Assumes code is built on your machine and mounted to this image. diff --git a/docker/datahub-ingestion/base-requirements.txt b/docker/datahub-ingestion/base-requirements.txt new file mode 100644 index 00000000000000..788baab9de5338 --- /dev/null +++ b/docker/datahub-ingestion/base-requirements.txt @@ -0,0 +1,314 @@ +acryl-datahub +absl-py==1.1.0 +acryl-iceberg-legacy==0.0.4 +acryl-PyHive==0.6.13 +aenum==3.1.11 +aiohttp==3.8.1 +aiosignal==1.2.0 +alembic==1.8.0 +altair==4.2.0 +anyio==3.6.1 +apache-airflow==2.2.5 +apache-airflow-providers-ftp==3.0.0 +apache-airflow-providers-http==3.0.0 +apache-airflow-providers-imap==3.0.0 +apache-airflow-providers-sqlite==3.0.0 +apispec==3.3.2 +argcomplete==2.0.0 +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +asgiref==3.5.2 +asn1crypto==1.5.1 +asttokens==2.0.5 +async-timeout==4.0.2 +attrs==20.3.0 +avro==1.10.2 +avro-gen3==0.7.4 +azure-core==1.24.1 +azure-identity==1.10.0 +azure-storage-blob==12.12.0 +azure-storage-file-datalake==12.7.0 +Babel==2.10.3 +backcall==0.2.0 +backports.zoneinfo==0.2.1 +beautifulsoup4==4.11.1 +bleach==5.0.0 +blinker==1.4 +boto3==1.24.14 +botocore==1.27.14 +bracex==2.3.post1 +cached-property==1.5.2 +cachelib==0.8.0 +cachetools==5.2.0 +cattrs==1.10.0 +certifi==2022.6.15 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.12 +click==8.1.3 +click-default-group==1.2.2 +clickclick==20.10.2 +clickhouse-driver==0.2.4 +clickhouse-sqlalchemy==0.1.8 +colorama==0.4.5 +colorlog==6.6.0 +commonmark==0.9.1 +confluent-kafka==1.8.2 +connexion==2.13.1 +croniter==1.3.5 +cryptography==36.0.2 +cx-Oracle==8.3.0 +debugpy==1.6.0 +decorator==5.1.1 +defusedxml==0.7.1 +Deprecated==1.2.13 +dill==0.3.5.1 +dnspython==2.2.1 +docker==5.0.3 +docutils==0.16 +ecdsa==0.17.0 +elasticsearch==7.13.4 +email-validator==1.2.1 +entrypoints==0.4 +et-xmlfile==1.1.0 +executing==0.8.3 +expandvars==0.9.0 +fastapi==0.78.0 +fastavro==1.5.1 +fastjsonschema==2.15.3 +feast==0.18.0 +Flask==1.1.2 +Flask-AppBuilder==3.4.5 +Flask-Babel==2.0.0 +Flask-Caching==1.11.1 +Flask-Cors==3.0.10 +Flask-JWT-Extended==3.25.1 +Flask-Login==0.4.1 +Flask-OpenID==1.3.0 +Flask-Session==0.4.0 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==0.14.3 +flatdict==4.0.1 +frozenlist==1.3.0 +future==0.18.2 +GeoAlchemy2==0.11.1 +google-api-core==2.8.0 +google-auth==2.8.0 +google-cloud-appengine-logging==1.1.2 +google-cloud-audit-log==0.2.0 +google-cloud-bigquery==3.2.0 +google-cloud-bigquery-storage==2.13.2 +google-cloud-core==2.3.1 +google-cloud-logging==3.1.1 +google-crc32c==1.3.0 +google-resumable-media==2.3.3 +googleapis-common-protos==1.52.0 +graphviz==0.20 +great-expectations==0.15.2 +greenlet==1.1.2 +grpc-google-iam-v1==0.12.3 +grpcio==1.44.0 +grpcio-reflection==1.44.0 +grpcio-status==1.44.0 +grpcio-tools==1.44.0 +gunicorn==20.1.0 +h11==0.12.0 +hmsclient==0.1.1 +httpcore==0.15.0 +httptools==0.4.0 +httpx==0.23.0 +humanfriendly==10.0 +idna==3.3 +ijson==3.1.4 +importlib-metadata==4.11.4 +importlib-resources==5.8.0 +inflection==0.5.1 +ipykernel==6.15.0 +ipython==8.4.0 +ipython-genutils==0.2.0 +iso8601==1.0.2 +isodate==0.6.1 +itsdangerous==1.1.0 +jedi==0.18.1 +Jinja2==3.0.3 +jmespath==1.0.1 +JPype1==1.4.0 +jsonlines==3.0.0 +jsonpatch==1.32 +jsonpointer==2.3 +jsonschema==3.2.0 +jupyter-client==7.3.4 +jupyter-core==4.10.0 +jupyterlab-pygments==0.2.2 +lazy-object-proxy==1.7.1 +linear-tsv==1.1.0 +lkml==1.2.0 +lockfile==0.12.2 +looker-sdk==22.2.1 +Mako==1.2.0 +Markdown==3.3.7 +MarkupSafe==2.0.1 +marshmallow==3.16.0 +marshmallow-enum==1.5.1 +marshmallow-oneofschema==3.0.1 +marshmallow-sqlalchemy==0.26.1 +matplotlib-inline==0.1.3 +mistune==0.8.4 +mixpanel==4.9.0 +mmh3==3.0.0 +more-itertools==8.13.0 +moto==3.1.14 +msal==1.16.0 +msal-extensions==1.0.0 +msrest==0.7.1 +multidict==6.0.2 +mypy-extensions==0.4.3 +nbclient==0.6.3 +nbconvert==6.5.0 +nbformat==5.4.0 +nest-asyncio==1.5.5 +networkx==2.8.4 +notebook==6.4.12 +numpy==1.22.4 +oauthlib==3.2.0 +okta==1.7.0 +openpyxl==3.0.10 +orderedset==2.0.3 +oscrypto==1.3.0 +packaging==21.3 +pandas==1.4.2 +pandavro==1.5.2 +pandocfilters==1.5.0 +parse==1.19.0 +parso==0.8.3 +pendulum==2.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +portalocker==2.4.0 +prison==0.2.1 +progressbar2==4.0.0 +prometheus-client==0.14.1 +prompt-toolkit==3.0.29 +proto-plus==1.19.6 +protobuf==3.20.1 +psutil==5.9.1 +psycopg2-binary==2.9.3 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pure-sasl==0.6.2 +py4j==0.10.9 +pyarrow==6.0.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pyathena==2.4.1 +pycparser==2.21 +pycryptodome==3.14.1 +pycryptodomex==3.14.1 +pydantic==1.9.1 +pydash==5.1.0 +pydeequ==1.0.1 +pydruid==0.6.3 +Pygments==2.12.0 +PyJWT==1.7.1 +pymongo==4.1.1 +PyMySQL==1.0.2 +pyOpenSSL==22.0.0 +pyparsing==2.4.7 +pyrsistent==0.18.1 +pyspark==3.0.3 +python-daemon==2.3.0 +python-dateutil==2.8.2 +python-dotenv==0.20.0 +python-jose==3.3.0 +python-ldap==3.4.0 +python-nvd3==0.15.0 +python-slugify==4.0.1 +python-tds==1.11.0 +python-utils==3.3.3 +python3-openid==3.2.0 +pytz==2022.1 +pytz-deprecation-shim==0.1.0.post0 +pytzdata==2020.1 +PyYAML==6.0 +pyzmq==23.2.0 +ratelimiter==1.2.0.post0 +redash-toolbelt==0.1.9 +requests==2.28.0 +requests-oauthlib==1.3.1 +responses==0.21.0 +retrying==1.3.3 +rfc3986==1.5.0 +rich==12.4.4 +rsa==4.8 +ruamel.yaml==0.17.17 +ruamel.yaml.clib==0.2.6 +s3transfer==0.6.0 +sasl3==0.2.11 +scipy==1.8.1 +Send2Trash==1.8.0 +setproctitle==1.2.3 +six==1.16.0 +smart-open==6.0.0 +sniffio==1.2.0 +snowflake-connector-python==2.7.8 +snowflake-sqlalchemy==1.2.4 +soupsieve==2.3.2.post1 +sql-metadata==2.2.2 +SQLAlchemy==1.3.24 +sqlalchemy-bigquery==1.4.4 +SQLAlchemy-JSONField==1.0.0 +sqlalchemy-pytds==0.3.4 +sqlalchemy-redshift==0.8.9 +SQLAlchemy-Utils==0.38.2 +sqlalchemy-vertica==0.0.5 +sqllineage==1.3.5 +sqlparse==0.4.2 +stack-data==0.3.0 +stackprinter==0.2.6 +starlette==0.19.1 +swagger-ui-bundle==0.0.9 +tableauserverclient==0.19.0 +tableschema==1.20.2 +tabulate==0.8.10 +tabulator==1.53.5 +tenacity==8.0.1 +tensorflow-metadata==1.9.0 +termcolor==1.1.0 +terminado==0.15.0 +text-unidecode==1.3 +thrift==0.16.0 +thrift-sasl==0.4.3 +tinycss2==1.1.1 +toml==0.10.2 +toolz==0.11.2 +tornado==6.1 +tqdm==4.64.0 +traitlets==5.2.1.post0 +trino==0.313.0 +types-Deprecated==1.2.8 +types-protobuf==3.19.22 +types-termcolor==1.1.4 +types-ujson==5.3.0 +typing-inspect==0.7.1 +typing_extensions==4.2.0 +tzdata==2022.1 +tzlocal==4.2 +ujson==5.3.0 +unicodecsv==0.14.1 +urllib3==1.26.9 +uvicorn==0.17.6 +uvloop==0.16.0 +vertica-python==1.1.0 +watchgod==0.8.2 +wcmatch==8.4 +wcwidth==0.2.5 +webencodings==0.5.1 +websocket-client==1.3.3 +websockets==10.3 +Werkzeug==1.0.1 +wrapt==1.14.1 +WTForms==2.3.3 +xlrd==2.0.1 +xmltodict==0.13.0 +yarl==1.7.2 +zipp==3.8.0 \ No newline at end of file diff --git a/docker/datahub-ingestion/base.Dockerfile b/docker/datahub-ingestion/base.Dockerfile new file mode 100644 index 00000000000000..dfffa64ea3a090 --- /dev/null +++ b/docker/datahub-ingestion/base.Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9.9 as base + +RUN apt-get update && apt-get install -y \ + jq \ + librdkafka-dev \ + python3-ldap \ + libldap2-dev \ + libsasl2-dev \ + libsasl2-modules \ + ldap-utils \ + && python -m pip install --upgrade pip wheel setuptools==57.5.0 + +COPY ./base-requirements.txt requirements.txt + +RUN pip install -r requirements.txt && \ + pip uninstall -y acryl-datahub \ No newline at end of file diff --git a/docker/datahub-ingestion/smoke.Dockerfile b/docker/datahub-ingestion/smoke.Dockerfile new file mode 100644 index 00000000000000..e406720083b59f --- /dev/null +++ b/docker/datahub-ingestion/smoke.Dockerfile @@ -0,0 +1,22 @@ +FROM acryldata/datahub-ingestion-base as base + +RUN apt-get update && apt-get install -y \ + sudo \ + python3-dev \ + libgtk2.0-0 \ + libgtk-3-0 \ + libgbm-dev \ + libnotify-dev \ + libgconf-2-4 \ + libnss3 \ + libxss1 \ + libasound2 \ + libxtst6 \ + xauth \ + xvfb + +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk + +COPY . /datahub-src +RUN cd /datahub-src && \ + ./gradlew :metadata-ingestion:installDev diff --git a/docker/datahub-mae-consumer/env/docker-without-neo4j.env b/docker/datahub-mae-consumer/env/docker-without-neo4j.env index 8c5d9729bee6c6..c71bf051b7be14 100644 --- a/docker/datahub-mae-consumer/env/docker-without-neo4j.env +++ b/docker/datahub-mae-consumer/env/docker-without-neo4j.env @@ -1,11 +1,12 @@ +DATAHUB_GMS_HOST=datahub-gms +DATAHUB_GMS_PORT=8080 + MAE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true KAFKA_BOOTSTRAP_SERVER=broker:29092 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 ELASTICSEARCH_HOST=elasticsearch ELASTICSEARCH_PORT=9200 -GMS_HOST=datahub-gms -GMS_PORT=8080 GRAPH_SERVICE_IMPL=elasticsearch ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml @@ -14,7 +15,7 @@ ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-regis # Uncomment to configure topic names # Make sure these names are consistent across the whole deployment -# KAFKA_TOPIC_NAME=MetadataAuditEvent_v4 +# METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 # DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 # Uncomment and set these to support SSL connection to Elasticsearch diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env index fba2625465de73..a26bd16be8ccb3 100644 --- a/docker/datahub-mae-consumer/env/docker.env +++ b/docker/datahub-mae-consumer/env/docker.env @@ -1,3 +1,6 @@ +DATAHUB_GMS_HOST=datahub-gms +DATAHUB_GMS_PORT=8080 + MAE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true KAFKA_BOOTSTRAP_SERVER=broker:29092 @@ -8,8 +11,6 @@ NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub -GMS_HOST=datahub-gms -GMS_PORT=8080 GRAPH_SERVICE_IMPL=neo4j ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml @@ -18,8 +19,12 @@ ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-regis # Uncomment to configure topic names # Make sure these names are consistent across the whole deployment -# KAFKA_TOPIC_NAME=MetadataAuditEvent_v4 +# METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 +# METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME=MetadataChangeLog_Timeseries_v1 +# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1 # DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 +# Deprecated! +# METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 # Uncomment and set these to support SSL connection to Elasticsearch # ELASTICSEARCH_USE_SSL= diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index ec47943a4ab95e..95d5dd2c789d1c 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -1,13 +1,16 @@ MCE_CONSUMER_ENABLED=true KAFKA_BOOTSTRAP_SERVER=broker:29092 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 -GMS_HOST=datahub-gms -GMS_PORT=8080 +DATAHUB_GMS_HOST=datahub-gms +DATAHUB_GMS_PORT=8080 # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment -# KAFKA_MCE_TOPIC_NAME=MetadataChangeEvent_v4 -# KAFKA_FMCE_TOPIC_NAME=FailedMetadataChangeEvent_v4 +# METADATA_CHANGE_PROPOSAL_TOPIC_NAME=MetadataChangeProposal_v1 +# FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME=FailedMetadataChangeProposal_v1 +# Deprecated! +# METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4 +# FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4 # Uncomment and set these to support SSL connection to GMS # NOTE: Currently GMS itself does not offer SSL support, these settings are intended for when there is a proxy in front diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile index 30c2e78feecaec..7b1604ecaa7eb2 100644 --- a/docker/datahub-upgrade/Dockerfile +++ b/docker/datahub-upgrade/Dockerfile @@ -1,16 +1,32 @@ # Defining environment ARG APP_ENV=prod -FROM adoptopenjdk/openjdk8:alpine-jre as base +FROM alpine:3.14 AS base ENV DOCKERIZE_VERSION v0.6.1 -RUN apk --no-cache add curl tar \ - && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv +# Upgrade Alpine and base packages +RUN apk --no-cache --update-cache --available upgrade \ + && if [ $(arch) = "aarch64" ]; then \ + DOCKERIZE_ARCH='aarch64';\ + elif [ $(arch) = "x86_64" ]; then \ + DOCKERIZE_ARCH='amd64'; \ + else \ + echo >&2 "Unsupported architecture $(arch)" ; exit 1; \ + fi \ + && apk --no-cache add tar curl openjdk8-jre bash coreutils gcompat \ + && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \ + && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \ + && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \ + && wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.4.1/opentelemetry-javaagent-all.jar \ + && wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -O jmx_prometheus_javaagent.jar \ + && cp /usr/lib/jvm/java-1.8-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks \ + && curl -L https://github.com/treff7es/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-${DOCKERIZE_ARCH}-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv + +FROM --platform=$BUILDPLATFORM alpine:3.14 AS prod-build + +# Upgrade Alpine and base packages +RUN apk --no-cache --update-cache --available upgrade \ + && apk --no-cache add openjdk8 perl -# Workaround alpine issue with /lib64 not being in the ld library path -# https://gitlab.alpinelinux.org/alpine/aports/-/issues/10140 -ENV LD_LIBRARY_PATH=/lib64 - -FROM openjdk:8 as prod-build COPY . datahub-src RUN cd datahub-src && ./gradlew :datahub-upgrade:build RUN cd datahub-src && cp datahub-upgrade/build/libs/datahub-upgrade.jar ../datahub-upgrade.jar diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 8c5fd123ee257b..c67b3afc9b1d90 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -33,7 +33,7 @@ services: kafka-setup: build: context: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} env_file: kafka-setup/env/docker.env hostname: kafka-setup container_name: kafka-setup @@ -85,7 +85,7 @@ services: build: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} env_file: elasticsearch-setup/env/docker.env hostname: elasticsearch-setup container_name: elasticsearch-setup @@ -120,7 +120,7 @@ services: build: context: ../ dockerfile: docker/datahub-gms/Dockerfile - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} env_file: ./datahub-gms/env/docker.cassandra.env hostname: datahub-gms container_name: datahub-gms @@ -136,7 +136,7 @@ services: build: context: ../ dockerfile: docker/datahub-frontend/Dockerfile - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} env_file: datahub-frontend/env/docker.env hostname: datahub-frontend-react container_name: datahub-frontend-react diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml index e9891abde199ce..e858555a1fc7db 100644 --- a/docker/docker-compose-without-neo4j.override.yml +++ b/docker/docker-compose-without-neo4j.override.yml @@ -17,7 +17,7 @@ services: build: context: ../ dockerfile: docker/mysql-setup/Dockerfile - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} env_file: mysql-setup/env/docker.env hostname: mysql-setup container_name: mysql-setup @@ -33,6 +33,8 @@ services: - mysql volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources + volumes: mysqldata: diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 481dee2be8ec79..d349d493639d05 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -31,7 +31,7 @@ services: kafka-setup: build: context: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} env_file: kafka-setup/env/docker.env hostname: kafka-setup container_name: kafka-setup @@ -72,7 +72,7 @@ services: build: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} env_file: elasticsearch-setup/env/docker.env hostname: elasticsearch-setup container_name: elasticsearch-setup @@ -83,7 +83,7 @@ services: build: context: ../ dockerfile: docker/datahub-gms/Dockerfile - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} env_file: datahub-gms/env/docker-without-neo4j.env hostname: datahub-gms container_name: datahub-gms @@ -98,7 +98,7 @@ services: build: context: ../ dockerfile: docker/datahub-frontend/Dockerfile - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} env_file: datahub-frontend/env/docker.env hostname: datahub-frontend-react container_name: datahub-frontend-react diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index e8b11c4e959fbc..7e3c056679b12d 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -48,6 +48,7 @@ services: - ../metadata-models/src/main/resources/:/datahub/datahub-gms/resources - ../metadata-service/war/build/libs/:/datahub/datahub-gms/bin - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources datahub-frontend-react: image: linkedin/datahub-frontend-react:debug diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index b6a095d8836222..ba9eb8ee2b3e9a 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -9,7 +9,7 @@ services: env_file: mysql/env/docker.env command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin ports: - - "3306:3306" + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - mysqldata:/var/lib/mysql @@ -18,7 +18,7 @@ services: build: context: ../ dockerfile: docker/mysql-setup/Dockerfile - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} env_file: mysql-setup/env/docker.env hostname: mysql-setup container_name: mysql-setup @@ -34,6 +34,6 @@ services: - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} volumes: - ${HOME}/.datahub/plugins/:/etc/datahub/plugins - + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources volumes: mysqldata: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index bf1ef52c7f6647..5165ec08222327 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,7 +12,7 @@ services: hostname: zookeeper container_name: zookeeper ports: - - "2181:2181" + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - zkdata:/var/opt/zookeeper @@ -24,8 +24,7 @@ services: depends_on: - zookeeper ports: - - "29092:29092" - - "9092:9092" + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: - broker:/var/lib/kafka/data/ @@ -33,7 +32,7 @@ services: kafka-setup: build: context: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} env_file: kafka-setup/env/docker.env hostname: kafka-setup container_name: kafka-setup @@ -50,7 +49,7 @@ services: - zookeeper - broker ports: - - "8081:8081" + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 elasticsearch: image: elasticsearch:7.9.3 @@ -58,7 +57,7 @@ services: container_name: elasticsearch hostname: elasticsearch ports: - - "9200:9200" + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 environment: - discovery.type=single-node - xpack.security.enabled=false @@ -75,8 +74,8 @@ services: hostname: neo4j container_name: neo4j ports: - - "7474:7474" - - "7687:7687" + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 volumes: - neo4jdata:/data @@ -85,7 +84,7 @@ services: build: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} env_file: elasticsearch-setup/env/docker.env hostname: elasticsearch-setup container_name: elasticsearch-setup @@ -96,11 +95,11 @@ services: build: context: ../ dockerfile: docker/datahub-gms/Dockerfile - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} hostname: datahub-gms container_name: datahub-gms ports: - - "8080:8080" + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 depends_on: - elasticsearch-setup - kafka-setup @@ -111,12 +110,12 @@ services: build: context: ../ dockerfile: docker/datahub-frontend/Dockerfile - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} env_file: datahub-frontend/env/docker.env hostname: datahub-frontend-react container_name: datahub-frontend-react ports: - - "9002:9002" + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 depends_on: - datahub-gms volumes: diff --git a/docker/elasticsearch-setup/create-indices.sh b/docker/elasticsearch-setup/create-indices.sh index 4ada7a7cb69626..94fe86e722014c 100755 --- a/docker/elasticsearch-setup/create-indices.sh +++ b/docker/elasticsearch-setup/create-indices.sh @@ -63,7 +63,7 @@ function create_datahub_usage_event_aws_elasticsearch() { fi if [ $(curl -o /dev/null -s -w "%{http_code}" --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_template/${PREFIX}datahub_usage_event_index_template") -eq 404 ] then - echo -e "\ncreating datahub_usagAe_event_index_template" + echo -e "\ncreating datahub_usage_event_index_template" sed -e "s/PREFIX/${PREFIX}/g" /index/usage-event/aws_es_index_template.json | tee -a /tmp/aws_es_index_template.json curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_template/${PREFIX}datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/tmp/aws_es_index_template.json curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/${PREFIX}datahub_usage_event-000001" -H 'Content-Type: application/json' --data "{\"aliases\":{\"${PREFIX}datahub_usage_event\":{\"is_write_index\":true}}}" diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh index a750a96eed8fe6..c4e0bdcb0975e1 100755 --- a/docker/kafka-setup/kafka-setup.sh +++ b/docker/kafka-setup/kafka-setup.sh @@ -11,17 +11,27 @@ CONNECTION_PROPERTIES_PATH=/tmp/connection.properties echo "bootstrap.servers=$KAFKA_BOOTSTRAP_SERVER" > $CONNECTION_PROPERTIES_PATH echo "security.protocol=$KAFKA_PROPERTIES_SECURITY_PROTOCOL" >> $CONNECTION_PROPERTIES_PATH +## Add support for SASL_PLAINTEXT +if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SASL_PLAINTEXT" ]]; then + echo "sasl.jaas.config=$KAFKA_PROPERTIES_SASL_JAAS_CONFIG" >> $CONNECTION_PROPERTIES_PATH + echo "sasl.kerberos.service.name=$KAFKA_PROPERTIES_SASL_KERBEROS_SERVICE_NAME" >> $CONNECTION_PROPERTIES_PATH +fi + if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SSL" ]]; then if [[ -n $KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION ]]; then echo "ssl.keystore.location=$KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH echo "ssl.keystore.password=$KAFKA_PROPERTIES_SSL_KEYSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - echo "ssl.keystore.type=$KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH echo "ssl.key.password=$KAFKA_PROPERTIES_SSL_KEY_PASSWORD" >> $CONNECTION_PROPERTIES_PATH + if [[ -n $KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE ]]; then + echo "ssl.keystore.type=$KAFKA_PROPERTIES_SSL_KEYSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH + fi fi if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION ]]; then echo "ssl.truststore.location=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION" >> $CONNECTION_PROPERTIES_PATH echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> $CONNECTION_PROPERTIES_PATH - echo "ssl.truststore.type=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH + if [[ -n $KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE ]]; then + echo "ssl.truststore.type=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE" >> $CONNECTION_PROPERTIES_PATH + fi fi echo "ssl.endpoint.identification.algorithm=$KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM" >> $CONNECTION_PROPERTIES_PATH fi @@ -42,3 +52,5 @@ kafka-topics.sh --create --if-not-exists --command-config $CONNECTION_PROPERTIES if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then kafka-topics.sh --create --if-not-exists --command-config $CONNECTION_PROPERTIES_PATH --bootstrap-server $KAFKA_BOOTSTRAP_SERVER --partitions $PARTITIONS --replication-factor $REPLICATION_FACTOR --topic $DATAHUB_USAGE_EVENT_NAME fi + +kafka-configs.sh --command-config $CONNECTION_PROPERTIES_PATH --bootstrap-server $KAFKA_BOOTSTRAP_SERVER --entity-type topics --entity-name _schemas --alter --add-config cleanup.policy=compact diff --git a/docker/mysql-setup/init.sql b/docker/mysql-setup/init.sql index 30b458ec87337e..6bd7133a359a89 100644 --- a/docker/mysql-setup/init.sql +++ b/docker/mysql-setup/init.sql @@ -16,6 +16,7 @@ create table if not exists metadata_aspect_v2 ( ); -- create default records for datahub user if not exists +DROP TABLE if exists temp_metadata_aspect_v2; CREATE TABLE temp_metadata_aspect_v2 LIKE metadata_aspect_v2; INSERT INTO temp_metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) VALUES( 'urn:li:corpuser:datahub', diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index ed3a2465a772f8..234ad5ee28c19a 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -15,16 +15,15 @@ services: - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m hostname: broker - image: kymeric/cp-kafka:latest + image: confluentinc/cp-kafka:7.2.0 ports: - - 29092:29092 - - 9092:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 datahub-actions: depends_on: - datahub-gms environment: - - GMS_HOST=datahub-gms - - GMS_PORT=8080 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 - KAFKA_BOOTSTRAP_SERVER=broker:29092 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 @@ -53,9 +52,9 @@ services: - ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - 9002:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: @@ -82,9 +81,9 @@ services: - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - PE_CONSUMER_ENABLED=true hostname: datahub-gms - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - 8080:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins elasticsearch: @@ -104,7 +103,7 @@ services: image: elasticsearch:7.9.3 mem_limit: 1g ports: - - 9200:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - esdata:/usr/share/elasticsearch/data elasticsearch-setup: @@ -116,7 +115,7 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} kafka-setup: container_name: kafka-setup depends_on: @@ -126,9 +125,9 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - MYSQL_DATABASE=datahub @@ -137,8 +136,9 @@ services: - MYSQL_ROOT_PASSWORD=datahub hostname: mysql image: mariadb:10.5.8 + # image: mysql:8 ports: - - 3306:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - mysqldata:/var/lib/mysql @@ -153,7 +153,7 @@ services: - MYSQL_PASSWORD=datahub - DATAHUB_DB_NAME=datahub hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} schema-registry: container_name: schema-registry depends_on: @@ -161,20 +161,20 @@ services: - broker environment: - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=PLAINTEXT://broker:29092 hostname: schema-registry - image: eugenetea/schema-registry-arm64:latest + image: confluentinc/cp-schema-registry:7.2.0 ports: - - 8081:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper - image: kymeric/cp-zookeeper:latest + image: confluentinc/cp-zookeeper:7.2.0 ports: - - 2181:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - zkdata:/var/opt/zookeeper version: '2.3' diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index a7ecacf2acf54a..ae6d660f8a8fcf 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -23,8 +23,8 @@ services: depends_on: - datahub-gms environment: - - GMS_HOST=datahub-gms - - GMS_PORT=8080 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 - KAFKA_BOOTSTRAP_SERVER=broker:29092 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 @@ -53,7 +53,7 @@ services: - ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - 9002:9002 volumes: @@ -82,13 +82,14 @@ services: - MCE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true - UI_INGESTION_ENABLED=true - - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35 + - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 hostname: datahub-gms - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - 8080:8080 volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources elasticsearch: container_name: elasticsearch environment: @@ -118,7 +119,7 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} kafka-setup: container_name: kafka-setup depends_on: @@ -128,7 +129,7 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin container_name: mysql @@ -155,7 +156,7 @@ services: - MYSQL_PASSWORD=datahub - DATAHUB_DB_NAME=datahub hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} schema-registry: container_name: schema-registry depends_on: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 5b84bec260b63b..93f6a359a73e7e 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -17,16 +17,15 @@ services: hostname: broker image: confluentinc/cp-kafka:5.4.0 ports: - - 29092:29092 - - 9092:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: - broker:/var/lib/kafka/data/ datahub-actions: depends_on: - datahub-gms environment: - - GMS_HOST=datahub-gms - - GMS_PORT=8080 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 - KAFKA_BOOTSTRAP_SERVER=broker:29092 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 @@ -55,9 +54,9 @@ services: - ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - 9002:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: @@ -88,13 +87,14 @@ services: - MCE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true - UI_INGESTION_ENABLED=true - - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35 + - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 hostname: datahub-gms - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - 8080:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - ${HOME}/.datahub/plugins/:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources elasticsearch: container_name: elasticsearch environment: @@ -112,7 +112,7 @@ services: image: elasticsearch:7.9.3 mem_limit: 1g ports: - - 9200:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - esdata:/usr/share/elasticsearch/data elasticsearch-setup: @@ -124,7 +124,7 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} ingest-api: container_name: ingest-api environment: @@ -152,7 +152,7 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin container_name: mysql @@ -164,7 +164,7 @@ services: hostname: mysql image: mysql:5.7 ports: - - 3306:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - mysqldata:/var/lib/mysql @@ -179,7 +179,7 @@ services: - MYSQL_PASSWORD=datahub - DATAHUB_DB_NAME=datahub hostname: mysql-setup - image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} neo4j: container_name: neo4j environment: @@ -189,8 +189,8 @@ services: hostname: neo4j image: neo4j:4.0.6 ports: - - 7474:7474 - - 7687:7687 + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 volumes: - neo4jdata:/data schema-registry: @@ -204,7 +204,7 @@ services: hostname: schema-registry image: confluentinc/cp-schema-registry:5.4.0 ports: - - 8081:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: @@ -213,7 +213,7 @@ services: hostname: zookeeper image: confluentinc/cp-zookeeper:5.4.0 ports: - - 2181:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - zkdata:/var/opt/zookeeper version: '2.3' diff --git a/docker/schema-registry/env/docker.env b/docker/schema-registry/env/docker.env index 166c551ac1f139..fbf6840116b9d7 100644 --- a/docker/schema-registry/env/docker.env +++ b/docker/schema-registry/env/docker.env @@ -1,2 +1,9 @@ SCHEMA_REGISTRY_HOST_NAME=schemaregistry SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + +# Uncomment to customize the Schema Registry kafka store connection +# SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT +# SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 +# ZOOKEEPER_SASL_ENABLED=false +# KAFKA_OPTS=-Xms1g -Xmx1g +# SCHEMA_REGISTRY_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false diff --git a/docs-website/build.gradle b/docs-website/build.gradle index aff51ab7a4e20f..a861df605d4659 100644 --- a/docs-website/build.gradle +++ b/docs-website/build.gradle @@ -12,7 +12,7 @@ node { } // Version of node to use. - version = '16.8.0' + version = '16.16.0' // Version of Yarn to use. yarnVersion = '1.22.0' @@ -124,6 +124,9 @@ clean { delete 'tmp' delete 'build' delete 'just' + delete fileTree(dir: 'genDocs', exclude: '.gitignore') + delete fileTree(dir: 'docs', exclude: '.gitignore') + delete 'graphql/combined.graphql' yarnClear } diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 43f116ae3ed24e..19e92e2c260316 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -6,9 +6,11 @@ module.exports = { onBrokenLinks: "throw", onBrokenMarkdownLinks: "throw", favicon: "img/favicon.ico", - organizationName: "linkedin", // Usually your GitHub org/user name. + organizationName: "datahub-project", // Usually your GitHub org/user name. projectName: "datahub", // Usually your repo name. - stylesheets: ["https://fonts.googleapis.com/css2?family=Manrope:wght@400;600&display=swap"], + stylesheets: [ + "https://fonts.googleapis.com/css2?family=Manrope:wght@400;600&display=swap", + ], themeConfig: { colorMode: { switchConfig: { @@ -170,7 +172,8 @@ module.exports = { additionalLanguages: ["ini"], }, algolia: { - apiKey: "26a4b687e96e7476b5a6f11365a83336", + appId: "RK0UG797F3", + apiKey: "39d7eb90d8b31d464e309375a52d674f", indexName: "datahubproject", // contextualSearch: true, // searchParameters: {}, @@ -198,7 +201,10 @@ module.exports = { ], ], plugins: [ - ["@docusaurus/plugin-ideal-image", { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }], + [ + "@docusaurus/plugin-ideal-image", + { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }, + ], "docusaurus-plugin-sass", [ "docusaurus-graphql-plugin", diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 1101a5238ba91d..ea945ee1f6f71f 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -49,6 +49,7 @@ module.exports = { "docs/components", "docs/architecture/metadata-ingestion", "docs/architecture/metadata-serving", + "docs/what/mxe", // "docs/what/gma", // "docs/what/gms", ], @@ -59,7 +60,55 @@ module.exports = { "docs/saas", "releases", ], - "Getting Started": ["docs/quickstart", "docs/cli", "docs/debugging"], + "Getting Started": ["docs/quickstart", "docs/debugging"], + Authentication: [ + { + type: "doc", + id: "docs/authentication/README", + label: "Overview", + }, + { + type: "doc", + id: "docs/authentication/concepts", + label: "Concepts", + }, + { + "Frontend Authentication": [ + "docs/authentication/guides/jaas", + { + "OIDC Authentication": [ + "docs/authentication/guides/sso/configure-oidc-react", + "docs/authentication/guides/sso/configure-oidc-react-google", + "docs/authentication/guides/sso/configure-oidc-react-okta", + "docs/authentication/guides/sso/configure-oidc-react-azure", + ], + }, + "docs/authentication/guides/add-users", + ], + }, + { + type: "doc", + id: "docs/authentication/introducing-metadata-service-authentication", + label: "Metadata Service Authentication", + }, + { + type: "doc", + id: "docs/authentication/personal-access-tokens", + label: "Personal Access Tokens", + }, + ], + Authorization: [ + { + type: "doc", + id: "docs/authorization/README", + label: "Overview", + }, + { + type: "doc", + id: "docs/authorization/policies", + label: "Access Policies", + }, + ], Ingestion: [ // add a custom label since the default is 'Metadata Ingestion' // note that we also have to add the path to this file in sidebarsjs_hardcoded_titles in generateDocsDir.ts @@ -139,6 +188,13 @@ module.exports = { // "docs/what/delta", // "docs/what/mxe", ], + CLI: [ + { + label: "Overview", + type: "doc", + id: "docs/cli", + }, + ], "GraphQL API": [ { label: "Overview", @@ -266,14 +322,13 @@ module.exports = { }, ], "Usage Guides": [ - "docs/policies", "docs/domains", "docs/ui-ingestion", "docs/tags", "docs/schema-history", "docs/how/search", - "docs/how/auth/add-users", "docs/how/ui-tabs-guide", + "docs/how/business-glossary-guide", ], "Developer Guides": [ // TODO: the titles of these should not be in question form in the sidebar @@ -288,15 +343,7 @@ module.exports = { //"docs/how/build-metadata-service", //"docs/how/graph-onboarding", //"docs/demo/graph-onboarding", - { - Authentication: [ - "docs/how/auth/jaas", - "docs/how/auth/sso/configure-oidc-react", - "docs/how/auth/sso/configure-oidc-react-google", - "docs/how/auth/sso/configure-oidc-react-okta", - "docs/how/auth/sso/configure-oidc-react-azure", - ], - }, + "docs/what/mxe", "docs/how/restore-indices", "docs/dev-guides/timeline", "docs/how/extract-container-logs", @@ -338,6 +385,7 @@ module.exports = { ], "Deployment Guides": [ "docs/how/kafka-config", + "datahub-ranger-plugin/README", "docker/README", "docs/deploy/kubernetes", "docker/datahub-upgrade/README", diff --git a/docs-website/src/components/Logos.js b/docs-website/src/components/Logos.js index b85d4b81247e6b..4e23fe4d38aaee 100644 --- a/docs-website/src/components/Logos.js +++ b/docs-website/src/components/Logos.js @@ -66,6 +66,11 @@ const companiesByIndustry = [ imageUrl: "/img/logos/companies/klarna.svg", size: "small", }, + { + name: "N26", + imageUrl: "/img/logos/companies/n26.svg", + size: "large", + }, { name: "BankSalad", imageUrl: "/img/logos/companies/banksalad.png", @@ -115,6 +120,11 @@ const companiesByIndustry = [ imageUrl: "/img/logos/companies/wolt.png", size: "large", }, + { + name: "Showroomprive.com", + imageUrl: "/img/logos/companies/showroomprive.png", + size: "default", + }, ], }, { @@ -125,6 +135,11 @@ const companiesByIndustry = [ imageUrl: "/img/logos/companies/cabify.png", size: "large", }, + { + name: "Digital Turbine", + imageUrl: "/img/logos/companies/digitalturbine.svg", + size: "defualt", + }, { name: "Viasat", imageUrl: "/img/logos/companies/viasat.png", diff --git a/docs-website/static/img/logos/companies/digitalturbine.svg b/docs-website/static/img/logos/companies/digitalturbine.svg new file mode 100644 index 00000000000000..54d6febbd455f2 --- /dev/null +++ b/docs-website/static/img/logos/companies/digitalturbine.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs-website/static/img/logos/companies/n26.svg b/docs-website/static/img/logos/companies/n26.svg new file mode 100644 index 00000000000000..8fde31ab2eb9df --- /dev/null +++ b/docs-website/static/img/logos/companies/n26.svg @@ -0,0 +1 @@ +N26_Logo_Black \ No newline at end of file diff --git a/docs-website/static/img/logos/companies/showroomprive.png b/docs-website/static/img/logos/companies/showroomprive.png new file mode 100644 index 00000000000000..dbf3c1fd15c0a3 Binary files /dev/null and b/docs-website/static/img/logos/companies/showroomprive.png differ diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index b44d8f08f401d2..2e4bb740768afb 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1977,6 +1977,46 @@ "@babel/runtime" "^7.7.2" regenerator-runtime "^0.13.3" +"@jridgewell/gen-mapping@^0.3.0": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" + integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A== + dependencies: + "@jridgewell/set-array" "^1.0.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.9" + +"@jridgewell/resolve-uri@^3.0.3": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" + integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== + +"@jridgewell/set-array@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== + +"@jridgewell/source-map@^0.3.2": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.2.tgz#f45351aaed4527a298512ec72f81040c998580fb" + integrity sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw== + dependencies: + "@jridgewell/gen-mapping" "^0.3.0" + "@jridgewell/trace-mapping" "^0.3.9" + +"@jridgewell/sourcemap-codec@^1.4.10": + version "1.4.14" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" + integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== + +"@jridgewell/trace-mapping@^0.3.9": + version "0.3.14" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz#b231a081d8f66796e475ad588a1ef473112701ed" + integrity sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ== + dependencies: + "@jridgewell/resolve-uri" "^3.0.3" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@mdx-js/mdx@1.6.22", "@mdx-js/mdx@^1.6.21": version "1.6.22" resolved "https://registry.yarnpkg.com/@mdx-js/mdx/-/mdx-1.6.22.tgz#8a723157bf90e78f17dc0f27995398e6c731f1ba" @@ -2610,10 +2650,10 @@ acorn@^6.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6" integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ== -acorn@^8.0.4, acorn@^8.4.1: - version "8.5.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.5.0.tgz#4512ccb99b3698c752591e9bb4472e38ad43cee2" - integrity sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q== +acorn@^8.0.4, acorn@^8.4.1, acorn@^8.5.0: + version "8.7.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" + integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== address@1.1.2, address@^1.0.1: version "1.1.2" @@ -8944,9 +8984,9 @@ source-map-resolve@^0.5.0: urix "^0.1.0" source-map-support@^0.5.17, source-map-support@~0.5.20: - version "0.5.20" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.20.tgz#12166089f8f5e5e8c56926b377633392dd2cb6c9" - integrity sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw== + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== dependencies: buffer-from "^1.0.0" source-map "^0.6.0" @@ -8966,11 +9006,6 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -source-map@~0.7.2: - version "0.7.3" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" - integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== - sourcemap-codec@^1.4.4: version "1.4.8" resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" @@ -9313,12 +9348,13 @@ terser-webpack-plugin@^5.1.3, terser-webpack-plugin@^5.2.4: terser "^5.7.2" terser@^5.7.2: - version "5.9.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.9.0.tgz#47d6e629a522963240f2b55fcaa3c99083d2c351" - integrity sha512-h5hxa23sCdpzcye/7b8YqbE5OwKca/ni0RQz1uRX3tGh8haaGHqcuSqbGRybuAKNdntZ0mDgFNXPJ48xQ2RXKQ== + version "5.14.2" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.14.2.tgz#9ac9f22b06994d736174f4091aa368db896f1c10" + integrity sha512-oL0rGeM/WFQCUd0y2QrWxYnq7tfSuKBiqTjRPWrRgB46WD/kiwHwF8T23z78H6Q6kGCuuHcPB+KULHRdxvVGQA== dependencies: + "@jridgewell/source-map" "^0.3.2" + acorn "^8.5.0" commander "^2.20.0" - source-map "~0.7.2" source-map-support "~0.5.20" text-table@0.2.0, text-table@^0.2.0: diff --git a/docs/actions/actions/executor.md b/docs/actions/actions/executor.md index 894e554328d6e7..784ac439408660 100644 --- a/docs/actions/actions/executor.md +++ b/docs/actions/actions/executor.md @@ -59,7 +59,7 @@ action: type: "executor" # Requires DataHub API configurations to report to DataHub datahub: - server: "http://${GMS_HOST:-localhost}:${GMS_PORT:-8080}" + server: "http://${DATAHUB_GMS_HOST:-localhost}:${DATAHUB_GMS_PORT:-8080}" # token: # Must have "Manage Secrets" privilege ``` diff --git a/docs/api/graphql/getting-started.md b/docs/api/graphql/getting-started.md index 393add57624814..6aae0b4e41d6c3 100644 --- a/docs/api/graphql/getting-started.md +++ b/docs/api/graphql/getting-started.md @@ -24,7 +24,7 @@ Today, DataHub's GraphQL endpoint is available for use in multiple places. The o 1. **Metadata Service**: The DataHub Metadata Service (backend) is the source-of-truth for the GraphQL endpoint. The endpoint is located at `/api/graphql` path of the DNS address where your instance of the `datahub-gms` container is deployed. For example, in local deployments it is typically located at `http://localhost:8080/api/graphql`. By default, -the Metadata Service has no explicit authentication checks. However, it does have *Authorization checks*. DataHub [Access Policies](../../../docs/policies.md) will be enforced by the GraphQL API. This means you'll need to provide an actor identity when querying the GraphQL API. +the Metadata Service has no explicit authentication checks. However, it does have *Authorization checks*. DataHub [Access Policies](../../authorization/policies.md) will be enforced by the GraphQL API. This means you'll need to provide an actor identity when querying the GraphQL API. To do so, include the `X-DataHub-Actor` header with an Authorized Corp User URN as the value in your request. Because anyone is able to set the value of this header, we recommend using this endpoint only in trusted environments, either by administrators themselves or programs that they own directly. 2. **Frontend Proxy**: The DataHub Frontend Proxy Service (frontend) is a basic web server & reverse proxy to the Metadata Service. As such, the diff --git a/docs/api/graphql/querying-entities.md b/docs/api/graphql/querying-entities.md index f38bca7c5707dd..b3ecd75d28048e 100644 --- a/docs/api/graphql/querying-entities.md +++ b/docs/api/graphql/querying-entities.md @@ -122,7 +122,7 @@ DataHub provides the following GraphQL mutations for updating entities in your M ### Authorization -Mutations which change Entity metadata are subject to [DataHub Access Policies](../../../docs/policies.md). This means that DataHub's server +Mutations which change Entity metadata are subject to [DataHub Access Policies](../../authorization/policies.md). This means that DataHub's server will check whether the requesting actor is authorized to perform the action. If you're querying the GraphQL endpoint via the DataHub Proxy Server, which is discussed more in [Getting Started](./getting-started.md), then the Session Cookie provided will carry the actor information. If you're querying the Metadata Service API directly, then you'll have to provide this via a special `X-DataHub-Actor` HTTP header, which should diff --git a/docs/architecture/metadata-ingestion.md b/docs/architecture/metadata-ingestion.md index 17177b083d0618..6ab8ab964c6b6c 100644 --- a/docs/architecture/metadata-ingestion.md +++ b/docs/architecture/metadata-ingestion.md @@ -8,10 +8,10 @@ DataHub supports an extremely flexible ingestion architecture that can support p The figure below describes all the options possible for connecting your favorite system to DataHub. ![Ingestion Architecture](../imgs/ingestion-architecture.png) -## MCE: The Center Piece +## Metadata Change Proposal: The Center Piece -The center piece for ingestion is the [Metadata Change Event (MCE)] which represents a metadata change that is being communicated by an upstream system. -MCE-s can be sent over Kafka, for highly scalable async publishing from source systems. They can also be sent directly to the HTTP endpoint exposed by the DataHub service tier to get synchronous success / failure responses. +The center piece for ingestion are [Metadata Change Proposals] which represent requests to make a metadata change to an organization's Metadata Graph. +Metadata Change Proposals can be sent over Kafka, for highly scalable async publishing from source systems. They can also be sent directly to the HTTP endpoint exposed by the DataHub service tier to get synchronous success / failure responses. ## Pull-based Integration @@ -19,17 +19,16 @@ DataHub ships with a Python based [metadata-ingestion system](../../metadata-ing ## Push-based Integration -As long as you can emit a [Metadata Change Event (MCE)] event to Kafka or make a REST call over HTTP, you can integrate any system with DataHub. For convenience, DataHub also provides simple [Python emitters] for you to integrate into your systems to emit metadata changes (MCE-s) at the point of origin. +As long as you can emit a [Metadata Change Proposal (MCP)] event to Kafka or make a REST call over HTTP, you can integrate any system with DataHub. For convenience, DataHub also provides simple [Python emitters] for you to integrate into your systems to emit metadata changes (MCP-s) at the point of origin. ## Internal Components -### Applying MCE-s to DataHub Service Tier (mce-consumer) +### Applying Metadata Change Proposals to DataHub Metadata Service (mce-consumer-job) -DataHub comes with a Kafka Streams based job, [mce-consumer-job], which consumes the MCE-s and converts them into the [equivalent Pegasus format] and sends it to the DataHub Service Tier (datahub-gms) using the `/ingest` endpoint. +DataHub comes with a Kafka Streams based job, [mce-consumer-job], which consumes the Metadata Change Proposals and writes them into the DataHub Metadata Service (datahub-gms) using the `/ingest` endpoint. -[Metadata Change Event (MCE)]: ../what/mxe.md#metadata-change-event-mce -[Metadata Audit Event (MAE)]: ../what/mxe.md#metadata-audit-event-mae -[MAE]: ../what/mxe.md#metadata-audit-event-mae +[Metadata Change Proposal (MCP)]: ../what/mxe.md#metadata-change-proposal-mcp +[Metadata Change Log (MCL)]: ../what/mxe.md#metadata-change-log-mcl [equivalent Pegasus format]: https://linkedin.github.io/rest.li/how_data_is_represented_in_memory#the-data-template-layer [mce-consumer-job]: ../../metadata-jobs/mce-consumer-job [Python emitters]: ../../metadata-ingestion/README.md#using-as-a-library diff --git a/docs/architecture/metadata-serving.md b/docs/architecture/metadata-serving.md index 3056790d29c5b3..3032848461b851 100644 --- a/docs/architecture/metadata-serving.md +++ b/docs/architecture/metadata-serving.md @@ -8,29 +8,28 @@ The figure below shows the high-level system diagram for DataHub's Serving Tier. ![datahub-serving](../imgs/datahub-serving.png) -The primary service is called [gms](../../metadata-service) and exposes a REST API and a GraphQL API for performing CRUD operations on metadata. The metadata service also exposes search and graph query API-s to support secondary-index style queries, full-text search queries as well as relationship queries like lineage. In addition, the [datahub-frontend](../../datahub-frontend) service expose a GraphQL API on top of the metadata graph. +The primary component is called [the Metadata Service](../../metadata-service) and exposes a REST API and a GraphQL API for performing CRUD operations on metadata. The service also exposes search and graph query API-s to support secondary-index style queries, full-text search queries as well as relationship queries like lineage. In addition, the [datahub-frontend](../../datahub-frontend) service expose a GraphQL API on top of the metadata graph. ## DataHub Serving Tier Components ### Metadata Storage -The DataHub Metadata Service (gms) persists metadata in a document store (could be an RDBMS like MySQL, Postgres or a key-value store like Couchbase etc.). +The DataHub Metadata Service persists metadata in a document store (an RDBMS like MySQL, Postgres, or Cassandra, etc.). -### Metadata Commit Log Stream (MAE) +### Metadata Change Log Stream (MCL) -The DataHub Service Tier also emits a commit event [Metadata Audit Event (MAE)] when a metadata change has been successfully committed to persistent storage. This event is sent over Kafka. +The DataHub Service Tier also emits a commit event [Metadata Change Log] when a metadata change has been successfully committed to persistent storage. This event is sent over Kafka. -The MAE stream is a public API and can be subscribed to by external systems providing an extremely powerful way to react in real-time to changes happening in metadata. For example, you could build an access control enforcer that reacts to change in metadata (e.g. a previously world-readable dataset now has a pii field) to immediately lock down the dataset in question. -Note that not all MCE-s will result in an MAE, because the DataHub serving tier will ignore any duplicate changes to metadata. +The MCL stream is a public API and can be subscribed to by external systems (for example, the Actions Framework) providing an extremely powerful way to react in real-time to changes happening in metadata. For example, you could build an access control enforcer that reacts to change in metadata (e.g. a previously world-readable dataset now has a pii field) to immediately lock down the dataset in question. +Note that not all MCP-s will result in an MCL, because the DataHub serving tier will ignore any duplicate changes to metadata. ### Metadata Index Applier (mae-consumer-job) -[MAE]-s are consumed by another Kafka Streams job, [mae-consumer-job], which applies the changes to the [graph] and [search index] accordingly. +[Metadata Change Logs]s are consumed by another Kafka Streams job, [mae-consumer-job], which applies the changes to the [graph] and [search index] accordingly. The job is entity-agnostic and will execute corresponding graph & search index builders, which will be invoked by the job when a specific metadata aspect is changed. The builder should instruct the job how to update the graph and search index based on the metadata change. -The builder can optionally use [Remote DAO] to fetch additional metadata from other sources to help compute the final update. -To ensure that metadata changes are processed in the correct chronological order, MAEs are keyed by the entity [URN] — meaning all MAEs for a particular entity will be processed sequentially by a single Kafka streams thread. +To ensure that metadata changes are processed in the correct chronological order, MCLs are keyed by the entity [URN] — meaning all MAEs for a particular entity will be processed sequentially by a single Kafka streams thread. ### Metadata Query Serving @@ -38,19 +37,20 @@ Primary-key based reads (e.g. getting schema metadata for a dataset based on the [RecordTemplate]: https://github.com/linkedin/rest.li/blob/master/data/src/main/java/com/linkedin/data/template/RecordTemplate.java [GenericRecord]: https://github.com/apache/avro/blob/master/lang/java/avro/src/main/java/org/apache/avro/generic/GenericRecord.java -[DAO]: https://en.wikipedia.org/wiki/Data_access_object [Pegasus]: https://linkedin.github.io/rest.li/DATA-Data-Schema-and-Templates [relationship]: ../what/relationship.md [entity]: ../what/entity.md [aspect]: ../what/aspect.md [GMS]: ../what/gms.md -[MAE]: ../what/mxe.md#metadata-audit-event-mae +[Metadata Change Log]: ../what/mxe.md#metadata-change-log-mcl [rest.li]: https://rest.li -[Metadata Change Event (MCE)]: ../what/mxe.md#metadata-change-event-mce -[Metadata Audit Event (MAE)]: ../what/mxe.md#metadata-audit-event-mae -[MAE]: ../what/mxe.md#metadata-audit-event-mae +[Metadata Change Proposal (MCP)]: ../what/mxe.md#metadata-change-proposal-mcp +[Metadata Change Log (MCL)]: ../what/mxe.md#metadata-change-log-mcl +[MCP]: ../what/mxe.md#metadata-change-proposal-mcp +[MCL]: ../what/mxe.md#metadata-change-log-mcl + [equivalent Pegasus format]: https://linkedin.github.io/rest.li/how_data_is_represented_in_memory#the-data-template-layer [graph]: ../what/graph.md [search index]: ../what/search-index.md diff --git a/docs/authentication/README.md b/docs/authentication/README.md new file mode 100644 index 00000000000000..9d2594e9fe238e --- /dev/null +++ b/docs/authentication/README.md @@ -0,0 +1,41 @@ +# Overview + +Authentication is the process of verifying the identity of a user or service. In DataHub this can be split into 2 main components: + - How to login into DataHub. + - How to make some action withing DataHub on **behalf** of a user/service. + +:::note + +Authentication in DataHub does not necessarily mean that the user/service being authenticated will be part of the metadata graph within DataHub itself other concepts like Datasets or Dashboards. +In other words, a user called `john.smith` logging into DataHub does not mean that john.smith appears as a CorpUser Entity within DataHub. + +For a quick video on that subject, have a look at our video on [DataHub Basics — Users, Groups, & Authentication 101 +](https://youtu.be/8Osw6p9vDYY) + +::: + +### Authentication in the Frontend + +Authentication in DataHub happens at 2 possible moments, if enabled. + +The first happens in the **DataHub Frontend** component when you access the UI. +You will be prompted with a login screen, upon which you must supply a username/password combo or OIDC login to access DataHub's UI. +This is typical scenario for a human interacting with DataHub. + +DataHub provides 2 methods of authentication: + - [JaaS Authentication](guides/jaas.md) for simple deployments where authenticated users are part of some known list or invited as a [Native DataHub User](guides/add-users.md). + - [OIDC Authentication](guides/sso/configure-oidc-react.md) to delegate authentication responsibility to third party systems like Okta or Google/Azure Authentication. This is the recommended approach for production systems. + +Upon validation of a user's credentials through one of these authentication systems, DataHub will generate a session token with which all subsequent requests will be made. + +### Authentication in the Backend + +The second way in which authentication occurs, is within DataHub's Backend (Metadata Service) when a user makes a request either through the UI or through APIs. +In this case DataHub makes use of Personal Access Tokens or session HTTP headers to apply actions on behalf of some user. +To learn more about DataHub's backend authentication have a look at our docs on [Introducing Metadata Service Authentication](introducing-metadata-service-authentication.md). + +Note, while authentication can happen on both the frontend or backend components of DataHub, they are separate, related processes. +The first is to authenticate users/services by a third party system (Open-ID connect or Java based authentication) and the latter to only permit identified requests to be accepted by DataHub via access tokens or bearer cookies. + +If you only want some users to interact with DataHub's UI, enable authentication in the Frontend and manage who is allowed either through JaaS or OIDC login methods. +If you want users to be able to access DataHub's backend directly without going through the UI in an authenticated manner, then enable authentication in the backend and generate access tokens for them. diff --git a/docs/authentication/concepts.md b/docs/authentication/concepts.md new file mode 100644 index 00000000000000..715e94c7e03808 --- /dev/null +++ b/docs/authentication/concepts.md @@ -0,0 +1,123 @@ +# Concepts & Key Components + +We introduced a few important concepts to the Metadata Service to make authentication work: + +1. Actor +2. Authenticator +3. AuthenticatorChain +4. AuthenticationFilter +5. DataHub Access Token +6. DataHub Token Service + +In following sections, we'll take a closer look at each individually. + +![](../imgs/metadata-service-auth.png) +*High level overview of Metadata Service Authentication* + +## What is an Actor? + +An **Actor** is a concept within the new Authentication subsystem to represent a unique identity / principal that is initiating actions (e.g. read & write requests) +on the platform. + +An actor can be characterized by 2 attributes: + +1. **Type**: The "type" of the actor making a request. The purpose is to for example distinguish between a "user" & "service" actor. Currently, the "user" actor type is the only one + formally supported. +2. **Id**: A unique identifier for the actor within DataHub. This is commonly known as a "principal" in other systems. In the case of users, this + represents a unique "username". This username is in turn used when converting from the "Actor" concept into a Metadata Entity Urn (e.g. CorpUserUrn). + +For example, the root "datahub" super user would have the following attributes: + +``` +{ + "type": "USER", + "id": "datahub" +} +``` + +Which is mapped to the CorpUser urn: + +``` +urn:li:corpuser:datahub +``` + +for Metadata retrieval. + +## What is an Authenticator? + +An **Authenticator** is a pluggable component inside the Metadata Service that is responsible for authenticating an inbound request provided context about the request (currently, the request headers). +Authentication boils down to successfully resolving an **Actor** to associate with the inbound request. + +There can be many types of Authenticator. For example, there can be Authenticators that + +- Verify the authenticity of access tokens (ie. issued by either DataHub itself or a 3rd-party IdP) +- Authenticate username / password credentials against a remote database (ie. LDAP) + +and more! A key goal of the abstraction is *extensibility*: a custom Authenticator can be developed to authenticate requests +based on an organization's unique needs. + +DataHub ships with 2 Authenticators by default: + +- **DataHubSystemAuthenticator**: Verifies that inbound requests have originated from inside DataHub itself using a shared system identifier + and secret. This authenticator is always present. + +- **DataHubTokenAuthenticator**: Verifies that inbound requests contain a DataHub-issued Access Token (discussed further in the "DataHub Access Token" section below) in their + 'Authorization' header. This authenticator is required if Metadata Service Authentication is enabled. + +## What is an AuthenticatorChain? + +An **AuthenticatorChain** is a series of **Authenticators** that are configured to run one-after-another. This allows +for configuring multiple ways to authenticate a given request, for example via LDAP OR via local key file. + +Only if each Authenticator within the chain fails to authenticate a request will it be rejected. + +The Authenticator Chain can be configured in the `application.yml` file under `authentication.authenticators`: + +``` +authentication: + .... + authenticators: + # Configure the Authenticators in the chain + - type: com.datahub.authentication.Authenticator1 + ... + - type: com.datahub.authentication.Authenticator2 + .... +``` + +## What is the AuthenticationFilter? + +The **AuthenticationFilter** is a [servlet filter](http://tutorials.jenkov.com/java-servlets/servlet-filters.html) that authenticates each and requests to the Metadata Service. +It does so by constructing and invoking an **AuthenticatorChain**, described above. + +If an Actor is unable to be resolved by the AuthenticatorChain, then a 401 unauthorized exception will be returned by the filter. + + +## What is a DataHub Token Service? What are Access Tokens? + +Along with Metadata Service Authentication comes an important new component called the **DataHub Token Service**. The purpose of this +component is twofold: + +1. Generate Access Tokens that grant access to the Metadata Service +2. Verify the validity of Access Tokens presented to the Metadata Service + +**Access Tokens** granted by the Token Service take the form of [Json Web Tokens](https://jwt.io/introduction), a type of stateless token which +has a finite lifespan & is verified using a unique signature. JWTs can also contain a set of claims embedded within them. Tokens issued by the Token +Service contain the following claims: + +- exp: the expiration time of the token +- version: version of the DataHub Access Token for purposes of evolvability (currently 1) +- type: The type of token, currently SESSION (used for UI-based sessions) or PERSONAL (used for personal access tokens) +- actorType: The type of the **Actor** associated with the token. Currently, USER is the only type supported. +- actorId: The id of the **Actor** associated with the token. + +Today, Access Tokens are granted by the Token Service under two scenarios: + +1. **UI Login**: When a user logs into the DataHub UI, for example via [JaaS](guides/jaas.md) or + [OIDC](guides/sso/configure-oidc-react.md), the `datahub-frontend` service issues an + request to the Metadata Service to generate a SESSION token *on behalf of* of the user logging in. (*Only the frontend service is authorized to perform this action). +2. **Generating Personal Access Tokens**: When a user requests to generate a Personal Access Token (described below) from the UI. + +> At present, the Token Service supports the symmetric signing method `HS256` to generate and verify tokens. + +Now that we're familiar with the concepts, we will talk concretely about what new capabilities have been built on top +of Metadata Service Authentication. \ No newline at end of file diff --git a/docs/how/auth/add-users.md b/docs/authentication/guides/add-users.md similarity index 76% rename from docs/how/auth/add-users.md rename to docs/authentication/guides/add-users.md index 3804b517e1dc76..5bd293425e8c7a 100644 --- a/docs/how/auth/add-users.md +++ b/docs/authentication/guides/add-users.md @@ -1,14 +1,49 @@ # Adding Users to DataHub -Users can log into DataHub in 2 ways: +Users can log into DataHub in 3 ways: -1. Static credentials (Simplest) -2. Single Sign-On via [OpenID Connect](https://www.google.com/search?q=openid+connect&oq=openid+connect&aqs=chrome.0.0i131i433i512j0i512l4j69i60l2j69i61.1468j0j7&sourceid=chrome&ie=UTF-8) (For Production Use) +1. Invite users via the UI +2. Static credentials +3. Single Sign-On via [OpenID Connect](https://www.google.com/search?q=openid+connect&oq=openid+connect&aqs=chrome.0.0i131i433i512j0i512l4j69i60l2j69i61.1468j0j7&sourceid=chrome&ie=UTF-8) (For Production Use) -which can be both enabled simultaneously. Option 1 is useful for running proof-of-concept exercises, or just getting DataHub up & running quickly. Option 2 is highly recommended for deploying DataHub in production. +which can be enabled simultaneously. Options 1 and 2 are useful for running proof-of-concept exercises, or just getting DataHub up & running quickly. Option 3 is highly recommended for deploying DataHub in production. +# Method 1: Inviting users via the DataHub UI -# Method 1: Configuring static credentials +## Send prospective users an invite link + +With the right permissions (`MANAGE_USER_CREDENTIALS`), you can invite new users to your deployed DataHub instance from the UI. It's as simple as sending a link! + +First navigate, to the Users and Groups tab (under Access) on the Settings page. You'll then see an `Invite Users` button. Note that this will only be clickable +if you have the correct permissions. + +![](../../imgs/invite-users-button.png) + +If you click on this button, you'll see a pop-up where you can copy an invite link to send to users, or generate a fresh one. + +![](../../imgs/invite-users-popup.png) + +When a new user visits the link, they will be directed to a sign up screen. Note that if a new link has since been regenerated, the new user won't be able to sign up! + +![](../../imgs/user-sign-up-screen.png) + +## Reset password for native users + +If a user forgets their password, an admin user with the `MANAGE_USER_CREDENTIALS` privilege can go to the Users and Groups tab and click on the respective user's +`Reset user password` button. + +![](../../imgs/reset-user-password-button.png) + +Similar to the invite link, you can generate a new reset link and send a link to that user which they can use to reset their credentials. + +![](../../imgs/reset-user-password-popup.png) + +When that user visits the link, they will be direct to a screen where they can reset their credentials. If the link is older than 24 hours or another link has since +been generated, they won't be able to reset their credentials! + +![](../../imgs/reset-credentials-screen.png) + +# Method 2: Configuring static credentials ## Create a user.props file @@ -134,8 +169,7 @@ and modify the `datahub-frontend-react` block to contain the extra volume mount. datahub docker quickstart —quickstart-compose-file .yml ``` - -# Method 2: Configuring SSO via OpenID Connect +# Method 3: Configuring SSO via OpenID Connect Setting up SSO via OpenID Connect means that users will be able to login to DataHub via a central Identity Provider such as @@ -148,7 +182,7 @@ Setting up SSO via OpenID Connect means that users will be able to login to Data and more. This option is recommended for production deployments of DataHub. For detailed information about configuring DataHub to use OIDC to -perform authentication, check out [OIDC Authentication](./sso/configure-oidc-react.md). +perform authentication, check out [OIDC Authentication](sso/configure-oidc-react.md). ## URNs @@ -159,7 +193,7 @@ when a user logs into DataHub via OIDC is used to construct a unique identifier urn:li:corpuser: ``` -For information about configuring which OIDC claim should be used as the username for Datahub, check out the [OIDC Authentication](./sso/configure-oidc-react.md) doc. +For information about configuring which OIDC claim should be used as the username for Datahub, check out the [OIDC Authentication](sso/configure-oidc-react.md) doc. ## FAQ diff --git a/docs/how/auth/jaas.md b/docs/authentication/guides/jaas.md similarity index 100% rename from docs/how/auth/jaas.md rename to docs/authentication/guides/jaas.md diff --git a/docs/how/auth/sso/configure-oidc-react-azure.md b/docs/authentication/guides/sso/configure-oidc-react-azure.md similarity index 100% rename from docs/how/auth/sso/configure-oidc-react-azure.md rename to docs/authentication/guides/sso/configure-oidc-react-azure.md diff --git a/docs/how/auth/sso/configure-oidc-react-google.md b/docs/authentication/guides/sso/configure-oidc-react-google.md similarity index 100% rename from docs/how/auth/sso/configure-oidc-react-google.md rename to docs/authentication/guides/sso/configure-oidc-react-google.md diff --git a/docs/how/auth/sso/configure-oidc-react-okta.md b/docs/authentication/guides/sso/configure-oidc-react-okta.md similarity index 100% rename from docs/how/auth/sso/configure-oidc-react-okta.md rename to docs/authentication/guides/sso/configure-oidc-react-okta.md diff --git a/docs/how/auth/sso/configure-oidc-react.md b/docs/authentication/guides/sso/configure-oidc-react.md similarity index 98% rename from docs/how/auth/sso/configure-oidc-react.md rename to docs/authentication/guides/sso/configure-oidc-react.md index 38560450162547..f33b6aa69971d8 100644 --- a/docs/how/auth/sso/configure-oidc-react.md +++ b/docs/authentication/guides/sso/configure-oidc-react.md @@ -1,4 +1,4 @@ -# OIDC Authentication +# Overview The DataHub React application supports OIDC authentication built on top of the [Pac4j Play](https://github.com/pac4j/play-pac4j) library. This enables operators of DataHub to integrate with 3rd party identity providers like Okta, Google, Keycloak, & more to authenticate their users. @@ -188,5 +188,4 @@ A brief summary of the steps that occur when the user navigates to the React app Even if OIDC is configured the root user can still login without OIDC by going to `/login` URL endpoint. It is recommended that you don't use the default credentials by mounting a different file in the front end container. To do this -please see [jaas](https://datahubproject.io/docs/how/auth/jaas/#mount-a-custom-userprops-file-docker-compose) - -"Mount a custom user.props file". +please see how to mount a custom user.props file for a JAAS authenticated deployment. diff --git a/docs/how/auth/sso/img/azure-setup-api-permissions.png b/docs/authentication/guides/sso/img/azure-setup-api-permissions.png similarity index 100% rename from docs/how/auth/sso/img/azure-setup-api-permissions.png rename to docs/authentication/guides/sso/img/azure-setup-api-permissions.png diff --git a/docs/how/auth/sso/img/azure-setup-app-registration.png b/docs/authentication/guides/sso/img/azure-setup-app-registration.png similarity index 100% rename from docs/how/auth/sso/img/azure-setup-app-registration.png rename to docs/authentication/guides/sso/img/azure-setup-app-registration.png diff --git a/docs/how/auth/sso/img/azure-setup-authentication.png b/docs/authentication/guides/sso/img/azure-setup-authentication.png similarity index 100% rename from docs/how/auth/sso/img/azure-setup-authentication.png rename to docs/authentication/guides/sso/img/azure-setup-authentication.png diff --git a/docs/how/auth/sso/img/azure-setup-certificates-secrets.png b/docs/authentication/guides/sso/img/azure-setup-certificates-secrets.png similarity index 100% rename from docs/how/auth/sso/img/azure-setup-certificates-secrets.png rename to docs/authentication/guides/sso/img/azure-setup-certificates-secrets.png diff --git a/docs/how/auth/sso/img/google-setup-1.png b/docs/authentication/guides/sso/img/google-setup-1.png similarity index 100% rename from docs/how/auth/sso/img/google-setup-1.png rename to docs/authentication/guides/sso/img/google-setup-1.png diff --git a/docs/how/auth/sso/img/google-setup-2.png b/docs/authentication/guides/sso/img/google-setup-2.png similarity index 100% rename from docs/how/auth/sso/img/google-setup-2.png rename to docs/authentication/guides/sso/img/google-setup-2.png diff --git a/docs/how/auth/sso/img/okta-setup-1.png b/docs/authentication/guides/sso/img/okta-setup-1.png similarity index 100% rename from docs/how/auth/sso/img/okta-setup-1.png rename to docs/authentication/guides/sso/img/okta-setup-1.png diff --git a/docs/how/auth/sso/img/okta-setup-2.png b/docs/authentication/guides/sso/img/okta-setup-2.png similarity index 100% rename from docs/how/auth/sso/img/okta-setup-2.png rename to docs/authentication/guides/sso/img/okta-setup-2.png diff --git a/docs/how/auth/sso/img/okta-setup-groups-claim.png b/docs/authentication/guides/sso/img/okta-setup-groups-claim.png similarity index 100% rename from docs/how/auth/sso/img/okta-setup-groups-claim.png rename to docs/authentication/guides/sso/img/okta-setup-groups-claim.png diff --git a/docs/introducing-metadata-service-authentication.md b/docs/authentication/introducing-metadata-service-authentication.md similarity index 56% rename from docs/introducing-metadata-service-authentication.md rename to docs/authentication/introducing-metadata-service-authentication.md index 31d4dee143b157..73ab1bdd568818 100644 --- a/docs/introducing-metadata-service-authentication.md +++ b/docs/authentication/introducing-metadata-service-authentication.md @@ -19,7 +19,7 @@ when a user navigated to `http://localhost:9002/`: b. If cookie was present + valid, redirect to the home page - c. If cookie was invalid, redirect to either a) the DataHub login screen (for [JAAS authentication](https://datahubproject.io/docs/how/auth/jaas/) or b) a [configured OIDC Identity Provider](https://datahubproject.io/docs/how/auth/sso/configure-oidc-react/) to perform authentication. + c. If cookie was invalid, redirect to either a) the DataHub login screen (for [JAAS authentication](guides/jaas.md) or b) a [configured OIDC Identity Provider](guides/sso/configure-oidc-react.md) to perform authentication. Once authentication had succeeded at the frontend proxy layer, a stateless (token-based) session cookie (PLAY_SESSION) would be set in the users browser. All subsequent requests, including the GraphQL requests issued by the React UI, would be authenticated using this session cookie. Once a request had made it beyond @@ -42,177 +42,9 @@ To address these problems, we introduced configurable Authentication inside the meaning that requests are no longer considered trusted until they are authenticated by the Metadata Service. Why push Authentication down? In addition to the problems described above, we wanted to plan for a future -where Authentication of Kafka-based-writes could be performed in the same manner as Rest writes. +where Authentication of Kafka-based-writes could be performed in the same manner as Rest writes. -Next, we'll cover the components being introduced to support Authentication inside the Metadata Service. - -### Concepts & Key Components - -We introduced a few important concepts to the Metadata Service to make authentication work: - -1. Actor -2. Authenticator -3. AuthenticatorChain -4. AuthenticationFilter -5. DataHub Access Token -6. DataHub Token Service - -In following sections, we'll take a closer look at each individually. - -![](./imgs/metadata-service-auth.png) -*High level overview of Metadata Service Authentication* - -#### What is an Actor? - -An **Actor** is a concept within the new Authentication subsystem to represent a unique identity / principal that is initiating actions (e.g. read & write requests) -on the platform. - -An actor can be characterized by 2 attributes: - -1. **Type**: The "type" of the actor making a request. The purpose is to for example distinguish between a "user" & "service" actor. Currently, the "user" actor type is the only one -formally supported. -2. **Id**: A unique identifier for the actor within DataHub. This is commonly known as a "principal" in other systems. In the case of users, this -represents a unique "username". This username is in turn used when converting from the "Actor" concept into a Metadata Entity Urn (e.g. CorpUserUrn). - -For example, the root "datahub" super user would have the following attributes: - -``` -{ - "type": "USER", - "id": "datahub" -} -``` - -Which is mapped to the CorpUser urn: - -``` -urn:li:corpuser:datahub -``` - -for Metadata retrieval. - -#### What is an Authenticator? - -An **Authenticator** is a pluggable component inside the Metadata Service that is responsible for authenticating an inbound request provided context about the request (currently, the request headers). -Authentication boils down to successfully resolving an **Actor** to associate with the inbound request. - -There can be many types of Authenticator. For example, there can be Authenticators that - -- Verify the authenticity of access tokens (ie. issued by either DataHub itself or a 3rd-party IdP) -- Authenticate username / password credentials against a remote database (ie. LDAP) - -and more! A key goal of the abstraction is *extensibility*: a custom Authenticator can be developed to authenticate requests -based on an organization's unique needs. - -DataHub ships with 2 Authenticators by default: - -- **DataHubSystemAuthenticator**: Verifies that inbound requests have originated from inside DataHub itself using a shared system identifier - and secret. This authenticator is always present. - -- **DataHubTokenAuthenticator**: Verifies that inbound requests contain a DataHub-issued Access Token (discussed further in the "DataHub Access Token" section below) in their -'Authorization' header. This authenticator is required if Metadata Service Authentication is enabled. - -#### What is an AuthenticatorChain? - -An **AuthenticatorChain** is a series of **Authenticators** that are configured to run one-after-another. This allows -for configuring multiple ways to authenticate a given request, for example via LDAP OR via local key file. - -Only if each Authenticator within the chain fails to authenticate a request will it be rejected. - -The Authenticator Chain can be configured in the `application.yml` file under `authentication.authenticators`: - -``` -authentication: - .... - authenticators: - # Configure the Authenticators in the chain - - type: com.datahub.authentication.Authenticator1 - ... - - type: com.datahub.authentication.Authenticator2 - .... -``` - -#### What is the AuthenticationFilter? - -The **AuthenticationFilter** is a [servlet filter](http://tutorials.jenkov.com/java-servlets/servlet-filters.html) that authenticates each and requests to the Metadata Service. -It does so by constructing and invoking an **AuthenticatorChain**, described above. - -If an Actor is unable to be resolved by the AuthenticatorChain, then a 401 unauthorized exception will be returned by the filter. - - -#### What is a DataHub Token Service? What are Access Tokens? - -Along with Metadata Service Authentication comes an important new component called the **DataHub Token Service**. The purpose of this -component is twofold: - -1. Generate Access Tokens that grant access to the Metadata Service -2. Verify the validity of Access Tokens presented to the Metadata Service - -**Access Tokens** granted by the Token Service take the form of [Json Web Tokens](https://jwt.io/introduction), a type of stateless token which -has a finite lifespan & is verified using a unique signature. JWTs can also contain a set of claims embedded within them. Tokens issued by the Token -Service contain the following claims: - -- exp: the expiration time of the token -- version: version of the DataHub Access Token for purposes of evolvability (currently 1) -- type: The type of token, currently SESSION (used for UI-based sessions) or PERSONAL (used for personal access tokens) -- actorType: The type of the **Actor** associated with the token. Currently, USER is the only type supported. -- actorId: The id of the **Actor** associated with the token. - -Today, Access Tokens are granted by the Token Service under two scenarios: - -1. **UI Login**: When a user logs into the DataHub UI, for example via [JaaS](https://datahubproject.io/docs/how/auth/jaas/) or - [OIDC](https://datahubproject.io/docs/how/auth/sso/configure-oidc-react/), the `datahub-frontend` service issues an - request to the Metadata Service to generate a SESSION token *on behalf of* of the user logging in. (*Only the frontend service is authorized to perform this action). -2. **Generating Personal Access Tokens**: When a user requests to generate a Personal Access Token (described below) from the UI. - -> At present, the Token Service supports the symmetric signing method `HS256` to generate and verify tokens. - -Now that we're familiar with the concepts, we will talk concretely about what new capabilities have been built on top -of Metadata Service Authentication. - -### New Capabilities - -#### Personal Access Tokens - -With these changes, we introduced a way to generate a "Personal Access Token" suitable for programmatic use with both the DataHub GraphQL -and DataHub Rest.li (Ingestion) APIs. - -Personal Access Tokens have a finite lifespan (default 3 months) and currently cannot be revoked without changing the signing key that -DataHub uses to generate these tokens (via the TokenService described above). Most importantly, they inherit the permissions -granted to the user who generates them. - -##### Generating Personal Access Tokens - -To generate a personal access token, users must have been granted the "Generate Personal Access Tokens" (GENERATE_PERSONAL_ACCESS_TOKENS) Privilege via a [DataHub Policy](./policies.md). Once -they have this permission, users can navigate to **'Settings'** > **'Access Tokens'** > **'Generate Personal Access Token'** to generate a token. - -![](./imgs/generate-personal-access-token.png) - -The token expiration dictates how long the token will be valid for. We recommend setting the shortest duration possible, as tokens are not currently -revokable once granted (without changing the signing key). - - -#### Using a Personal Access Token - -The user will subsequently be able to make authenticated requests to DataHub frontend proxy or DataHub GMS directly by providing -the generated Access Token as a Bearer token in the `Authorization` header: - -``` -Authorization: Bearer -``` - -For example, using a curl to the frontend proxy (preferred in production): - -`curl 'http://localhost:9002/api/gms/entities/urn:li:corpuser:datahub' -H 'Authorization: Bearer ` - -or to Metadata Service directly: - -`curl 'http://localhost:8080/entities/urn:li:corpuser:datahub' -H 'Authorization: Bearer ` - -Without an access token, making programmatic requests will result in a 401 result from the server if Metadata Service Authentication -is enabled. - -### Configuring Metadata Service Authentication +## Configuring Metadata Service Authentication Metadata Service Authentication is currently **opt-in**. This means that you may continue to use DataHub without Metadata Service Authentication without interruption. To enable Metadata Service Authentication: @@ -238,7 +70,7 @@ contains a valid Access Token for the Metadata Service. When browsing the UI, th to authenticate each request. For users who want to access the Metadata Service programmatically, i.e. for running ingestion, the current recommendation is to generate -a **Personal Access Token** (described above) from the root "datahub" user account, and using this token when configuring your [Ingestion Recipes](https://datahubproject.io/docs/metadata-ingestion/#recipes). +a **Personal Access Token** (described above) from the root "datahub" user account, and using this token when configuring your [Ingestion Recipes](../../metadata-ingestion/README.md#recipes). To configure the token for use in ingestion, simply populate the "token" configuration for the `datahub-rest` sink: ``` @@ -339,7 +171,7 @@ to the **DataHub Frontend Proxy**, as routing to Metadata Service endpoints is c This recommendation is in effort to minimize the exposed surface area of DataHub to make securing, operating, maintaining, and developing the platform simpler. -In practice, this will require migrating Metadata [Ingestion Recipes](https://datahubproject.io/docs/metadata-ingestion/#recipes) use the `datahub-rest` sink to pointing at a slightly different +In practice, this will require migrating Metadata [Ingestion Recipes](../../metadata-ingestion/README.md#recipes) use the `datahub-rest` sink to pointing at a slightly different host + path. Example recipe that proxies through DataHub Frontend diff --git a/docs/authentication/personal-access-tokens.md b/docs/authentication/personal-access-tokens.md new file mode 100644 index 00000000000000..a5cf8ad15ac3f5 --- /dev/null +++ b/docs/authentication/personal-access-tokens.md @@ -0,0 +1,47 @@ +# Personal Access Tokens + +With these changes, we introduced a way to generate a "Personal Access Token" suitable for programmatic use with both the DataHub GraphQL +and DataHub Rest.li (Ingestion) APIs. + +Personal Access Tokens have a finite lifespan (default 3 months) and currently cannot be revoked without changing the signing key that +DataHub uses to generate these tokens (via the TokenService described above). Most importantly, they inherit the permissions +granted to the user who generates them. + +## Generating Personal Access Tokens + +To generate a personal access token, users must have been granted the "Generate Personal Access Tokens" (GENERATE_PERSONAL_ACCESS_TOKENS) or "Manage All Access Tokens" Privilege via a [DataHub Policy](../authorization/policies.md). Once +they have this permission, users can navigate to **'Settings'** > **'Access Tokens'** > **'Generate Personal Access Token'** to generate a token. + +![](../imgs/generate-personal-access-token.png) + +The token expiration dictates how long the token will be valid for. We recommend setting the shortest duration possible, as tokens are not currently +revokable once granted (without changing the signing key). + + +## Using Personal Access Tokens + +The user will subsequently be able to make authenticated requests to DataHub frontend proxy or DataHub GMS directly by providing +the generated Access Token as a Bearer token in the `Authorization` header: + +``` +Authorization: Bearer +``` + +For example, using a curl to the frontend proxy (preferred in production): + +`curl 'http://localhost:9002/api/gms/entities/urn:li:corpuser:datahub' -H 'Authorization: Bearer ` + +or to Metadata Service directly: + +`curl 'http://localhost:8080/entities/urn:li:corpuser:datahub' -H 'Authorization: Bearer ` + +Since authorization now happens at the GMS level, this means that ingestion is also protected behind access tokens, to use them simply add a `token` to the sink config property as seen below: + +![](../imgs/ingestion-with-token.png) + +:::note + +Without an access token, making programmatic requests will result in a 401 result from the server if Metadata Service Authentication +is enabled. + +::: diff --git a/docs/authorization/README.md b/docs/authorization/README.md new file mode 100644 index 00000000000000..60eda3ca3147a4 --- /dev/null +++ b/docs/authorization/README.md @@ -0,0 +1,18 @@ +# Overview + +Authorization specifies _what_ accesses an _authenticated_ user has within a system. +This section is all about how DataHub authorizes a given user/service that wants to interact with the system. + +:::note + +Authorization only makes sense in the context of an **Authenticated** DataHub deployment. To use DataHub's authorization features +please first make sure that the system has been configured from an authentication perspective as you intend. + +::: + +Once the identity of a user or service has been established, DataHub determines what accesses the authenticated request has. + +This is done by checking what operation a given user/service wants to perform within DataHub & whether it is allowed to do so. +The set of operations that are allowed in DataHub are what we call **Policies**. + +Policies specify fine-grain access control for _who_ can do _what_ to _which_ resources, for more details on the set of Policies that DataHub provides please see the [Policies Guide](../authorization/policies.md). diff --git a/docs/policies.md b/docs/authorization/policies.md similarity index 93% rename from docs/policies.md rename to docs/authorization/policies.md index 2e0a1f972bce46..32a89aca49384f 100644 --- a/docs/policies.md +++ b/docs/authorization/policies.md @@ -75,11 +75,16 @@ We currently support the following: | Manage Policies | Allow actor to create and remove access control policies. Be careful - Actors with this privilege are effectively super users. | | Manage Metadata Ingestion | Allow actor to create, remove, and update Metadata Ingestion sources. | | Manage Secrets | Allow actor to create & remove secrets stored inside DataHub. | -| Manage Users & Groups | Allow actor to create, remove, and update users and groups on DataHub. | +| Manage Users & Groups | Allow actor to create, remove, and update users and groups on DataHub. | | Manage All Access Tokens | Allow actor to create, remove, and list access tokens for all users on DataHub. | -| Manage Domains | Allow actor to create and remove Asset Domains. | -| View Analytics | Allow the actor access to the DataHub analytics dashboard. | +| Create Domains | Allow the actor to create new Domains | +| Manage Domains | Allow actor to create and remove any Domains. | +| View Analytics | Allow the actor access to the DataHub analytics dashboard. | | Generate Personal Access Tokens | Allow the actor to generate access tokens for personal use with DataHub APIs. | +| Manage User Credentials | Allow the actor to generate invite links for new native DataHub users, and password reset links for existing native users. | +| Manage Glossaries | Allow the actor to create, edit, move, and delete Glossary Terms and Term Groups | +| Create Tags | Allow the actor to create new Tags | +| Manage Tags | Allow the actor to create and remove any Tags | **Common metadata privileges** to view & modify any entity within DataHub. diff --git a/docs/cli.md b/docs/cli.md index 5a44c27c9ae207..83268b9ed9b41d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -129,7 +129,10 @@ datahub check plugins ## Environment variables supported The env variables take precedence over what is in the DataHub CLI config created through `init` command. The list of supported environment variables are as follows - `DATAHUB_SKIP_CONFIG` (default `false`) - Set to `true` to skip creating the configuration file. -- `DATAHUB_GMS_HOST` (default `http://localhost:8080`) - Set to a URL of GMS instance. +- `DATAHUB_GMS_URL` (default `http://localhost:8080`) - Set to a URL of GMS instance +- `DATAHUB_GMS_HOST` (default `localhost`) - Set to a host of GMS instance. Prefer using `DATAHUB_GMS_URL` to set the URL. +- `DATAHUB_GMS_PORT` (default `8080`) - Set to a port of GMS instance. Prefer using `DATAHUB_GMS_URL` to set the URL. +- `DATAHUB_GMS_PROTOCOL` (default `http`) - Set to a protocol like `http` or `https`. Prefer using `DATAHUB_GMS_URL` to set the URL. - `DATAHUB_GMS_TOKEN` (default `None`) - Used for communicating with DataHub Cloud. - `DATAHUB_TELEMETRY_ENABLED` (default `true`) - Set to `false` to disable telemetry. If CLI is being run in an environment with no access to public internet then this should be disabled. - `DATAHUB_TELEMETRY_TIMEOUT` (default `10`) - Set to a custom integer value to specify timeout in secs when sending telemetry. @@ -139,7 +142,7 @@ The env variables take precedence over what is in the DataHub CLI config created ```shell DATAHUB_SKIP_CONFIG=false -DATAHUB_GMS_HOST=http://localhost:8080 +DATAHUB_GMS_URL=http://localhost:8080 DATAHUB_GMS_TOKEN= DATAHUB_TELEMETRY_ENABLED=true DATAHUB_TELEMETRY_TIMEOUT=10 diff --git a/docs/debugging.md b/docs/debugging.md index 3697ee508d48ed..5396423ce1e3ba 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -205,3 +205,18 @@ You'll need to ingest some metadata of the following form to see it inside the D "proposedDelta": null } ``` + +## I've configured OIDC, but I cannot login. I get continuously redirected. What do I do? + +Sorry to hear that! + +This phenomena may be due to the size of a Cookie DataHub uses to authenticate its users. If it's too large ( > 4096), then you'll see this behavior. The cookie embeds an encoded version of the information returned by your OIDC Identity Provider - if they return a lot of information, this can be the root cause. + +One solution is to use Play Cache to persist this session information for a user. This means the attributes about the user (and their session info) will be stored in an in-memory store in the `datahub-frontend` service, instead of a browser-side cookie. + +To configure the Play Cache session store, you can set the env variable "PAC4J_SESSIONSTORE_PROVIDER" as "PlayCacheSessionStore" for the `datahub-frontend` container. + +Do note that there are downsides to using the Play Cache. Specifically, it will make `datahub-frontend` a stateful server. If you have multiple instances of `datahub-frontend` deployed, you'll need to ensure that the same user is deterministically routed to the same service container (since the sessions are stored in memory). If you're using a single instance of `datahub-frontend` (the default), then things should "just work". + +For more details, please refer to https://github.com/datahub-project/datahub/pull/5114 + diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index ff9d7e6345768d..13af8f89d19ba0 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -162,7 +162,7 @@ You need to request a certificate in the AWS Certificate Manager by following th the ARN of the new certificate. You also need to replace host-name with the hostname of choice like demo.datahubproject.io. -To have the metadata [authentication service](https://datahubproject.io/docs/introducing-metadata-service-authentication/#configuring-metadata-service-authentication) enable and use [API tokens](https://datahubproject.io/docs/introducing-metadata-service-authentication/#generating-personal-access-tokens) from the UI you will need to set the configuration in the values.yaml for the `gms` and the `frontend` deployments. This could be done by enabling the `metadata_service_authentication`: +To have the metadata [authentication service](../authentication/introducing-metadata-service-authentication.md#Configuring Metadata Service Authentication) enabled and use [API tokens](../authentication/personal-access-tokens.md#Generating Personal Access Tokens) from the UI you will need to set the configuration in the values.yaml for the `gms` and the `frontend` deployments. This could be done by enabling the `metadata_service_authentication`: ``` datahub: diff --git a/docs/deploy/confluent-cloud.md b/docs/deploy/confluent-cloud.md index 32e8810f41d280..70ef04f2131fc3 100644 --- a/docs/deploy/confluent-cloud.md +++ b/docs/deploy/confluent-cloud.md @@ -2,24 +2,26 @@ DataHub provides the ability to easily leverage Confluent Cloud as your Kafka provider. To do so, you'll need to configure DataHub to talk to a broker and schema registry hosted by Confluent. -Doing this is a matter of configuring the Kafka Producer and Consumers used by DataHub correctly. There are 2 places where Kafka configuration should be provided: the metadata server (GMS) and the frontend server (datahub-frontend). Follow the steps below to configure these components for your deployment. +Doing this is a matter of configuring the Kafka Producer and Consumers used by DataHub correctly. There are 2 places where Kafka configuration should be provided: the metadata service (GMS) and the frontend server (datahub-frontend). Follow the steps below to configure these components for your deployment. ## **Step 1: Create topics in Confluent Control Center** First, you'll need to create following new topics in the [Confluent Control Center](https://docs.confluent.io/platform/current/control-center/index.html). By default they have the following names: -1. **MetadataChangeEvent_v4**: Metadata change proposal messages -2. **MetadataAuditEvent_v4**: Metadata change log messages -3. **FailedMetadataChangeEvent_v4**: Failed to process #1 event -4. **DataHubUsageEvent_v1**: User behavior tracking event for UI -5. **MetadataChangeProposal_v1** -6. **FailedMetadataChangeProposal_v1** -7. **MetadataChangeLog_Versioned_v1** -8. **MetadataChangeLog_Timeseries_v1** +1. **MetadataChangeProposal_v1** +2. **FailedMetadataChangeProposal_v1** +3. **MetadataChangeLog_Versioned_v1** +4. **MetadataChangeLog_Timeseries_v1** +5. **DataHubUsageEvent_v1**: User behavior tracking event for UI +6. (Deprecated) **MetadataChangeEvent_v4**: Metadata change proposal messages +7. (Deprecated) **MetadataAuditEvent_v4**: Metadata change log messages +8. (Deprecated) **FailedMetadataChangeEvent_v4**: Failed to process #1 event -The last 4 are exaplined in [MCP/MCL](../advanced/mcp-mcl.md) +The first five are the most important, and are explained in more depth in [MCP/MCL](../advanced/mcp-mcl.md). The final topics are +those which are deprecated but still used under certain circumstances. It is likely that in the future they will be completely +decommissioned. -To do so, navigate to your **Cluster** and click "Create Topic". Feel free to tweak the default topic configurations to +To create the topics, navigate to your **Cluster** and click "Create Topic". Feel free to tweak the default topic configurations to match your preferences. ![CreateTopic](../imgs/confluent-create-topic.png) @@ -59,13 +61,14 @@ KAFKA_PROPERTIES_BASIC_AUTH_CREDENTIALS_SOURCE=USER_INFO KAFKA_PROPERTIES_BASIC_AUTH_USER_INFO=P2ETAN5QR2LCWL14:RTjqw7AfETDl0RZo/7R0123LhPYs2TGjFKmvMWUFnlJ3uKubFbB1Sfs7aOjjNi1m23 ``` -Note that this step is only required if DATAHUB_ANALYTICS_ENABLED is not set to false. +Note that this step is only required if `DATAHUB_ANALYTICS_ENABLED` environment variable is not explicitly set to false for the datahub-frontend +container. If you're deploying with Docker Compose, you do not need to deploy the Zookeeper, Kafka Broker, or Schema Registry containers that ship by default. ### Helm -If you're deploying to K8s using Helm, you can simply change the `datahub-helm` values.yml to point to Confluent Cloud and disable some default containers: +If you're deploying on K8s using Helm, you can simply change the **datahub-helm** `values.yml` to point to Confluent Cloud and disable some default containers: First, disable the `cp-schema-registry` service: @@ -106,7 +109,7 @@ automatically populate with your new secrets: You'll need to copy the values of `sasl.jaas.config` and `basic.auth.user.info` for the next step. -The next step is to create K8s secrets containing the config values you've just generated. Specifically, you'll run the following commands: +The next step is to create K8s secrets containing the config values you've just generated. Specifically, you can run the following commands: ```shell kubectl create secret generic confluent-secrets --from-literal=sasl_jaas_config="" @@ -120,7 +123,7 @@ kubectl create secret generic confluent-secrets --from-literal=sasl_jaas_config= kubectl create secret generic confluent-secrets --from-literal=basic_auth_user_info="P2ETAN5QR2LCWL14:RTjqw7AfETDl0RZo/7R0123LhPYs2TGjFKmvMWUFnlJ3uKubFbB1Sfs7aOjjNi1m23" ``` -Finally, we'll configure our containers to pick up the Confluent Kafka Configs by changing two config blocks in our values.yaml file. You +Finally, we'll configure our containers to pick up the Confluent Kafka Configs by changing two config blocks in our `values.yaml` file. You should see these blocks commented at the bottom of the template. You'll want to uncomment them and set them to the following values: ``` @@ -132,7 +135,7 @@ credentialsAndCertsSecrets: springKafkaConfigurationOverrides: - security.protocol: SASL + security.protocol: SASL_SSL sasl.mechanism: PLAIN client.dns.lookup: use_all_dns_ips basic.auth.credentials.source: USER_INFO @@ -143,6 +146,6 @@ Then simply apply the updated `values.yaml` to your K8s cluster via `kubectl app ## Contribution Accepting contributions for a setup script compatible with Confluent Cloud! -Currently the kafka-setup-job container we ship with is only compatible with a distribution of Kafka wherein ZooKeeper +The kafka-setup-job container we ship with is only compatible with a distribution of Kafka wherein ZooKeeper is exposed and available. A version of the job using the [Confluent CLI](https://docs.confluent.io/confluent-cli/current/command-reference/kafka/topic/confluent_kafka_topic_create.html) would be very useful for the broader community. \ No newline at end of file diff --git a/docs/domains.md b/docs/domains.md index 700f7a719eda5f..436f7ba1f332a5 100644 --- a/docs/domains.md +++ b/docs/domains.md @@ -18,7 +18,7 @@ DataHub supports Tags, Glossary Terms, & Domains as distinct types of Metadata t ## Creating a Domain To create a Domain, first navigate to the **Domains** tab in the top-right menu of DataHub. Users must have the Platform Privilege -called `Manage Domains` to view this tab, which can be granted by creating a new Platform [Policy](./policies.md). +called `Manage Domains` to view this tab, which can be granted by creating a new Platform [Policy](authorization/policies.md). ![](./imgs/domains-tab.png) @@ -47,8 +47,11 @@ By default, you don't need to worry about this. DataHub will auto-generate an un Once you've chosen a name and a description, click 'Create' to create the new Domain. -## Assigning an Asset to a Domain +## Assigning an Asset to a Domain +You can assign assets to Domain using the UI or programmatically using the API or during ingestion. + +### UI-Based Assignment To assign an asset to a Domain, simply navigate to the asset's profile page. At the bottom left-side menu bar, you'll see a 'Domain' section. Click 'Set Domain', and then search for the Domain you'd like to add to. When you're done, click 'Add'. @@ -57,7 +60,81 @@ see a 'Domain' section. Click 'Set Domain', and then search for the Domain you'd To remove an asset from a Domain, click the 'x' icon on the Domain tag. > Notice: Adding or removing an asset from a Domain requires the `Edit Domain` Metadata Privilege, which can be granted -> by a [Policy](./policies.md). +> by a [Policy](authorization/policies.md). + +### Ingestion-time Assignment +All SQL-based ingestion sources support assigning domains during ingestion using the `domain` configuration. Consult your source's configuration details page (e.g. [Snowflake](./generated/ingestion/sources/snowflake.md)), to verify that it supports the Domain capability. + +:::note + +Assignment of domains during ingestion will overwrite domains that you have assigned in the UI. A single table can only belong to one domain. + +::: + + +Here is a quick example of a snowflake ingestion recipe that has been enhanced to attach the **Analytics** domain to all tables in the **long_tail_companions** database in the **analytics** schema, and the **Finance** domain to all tables in the **long_tail_companions** database in the **ecommerce** schema. + +```yaml +source: + type: snowflake + config: + username: ${SNOW_USER} + password: ${SNOW_PASS} + account_id: + warehouse: COMPUTE_WH + role: accountadmin + database_pattern: + allow: + - "long_tail_companions" + schema_pattern: + deny: + - information_schema + profiling: + enabled: False + domain: + Analytics: + allow: + - "long_tail_companions.analytics.*" + Finance: + allow: + - "long_tail_companions.ecommerce.*" +``` + +:::note + +When bare domain names like `Analytics` is used, the ingestion system will first check if a domain like `urn:li:domain:Analytics` is provisioned, failing that; it will check for a provisioned domain that has the same name. If we are unable to resolve bare domain names to provisioned domains, then ingestion will refuse to proceeed until the domain is provisioned on DataHub. + +::: + +You can also provide fully-qualified domain names to ensure that no ingestion-time domain resolution is needed. For example, the following recipe shows an example using fully qualified domain names: + +```yaml +source: + type: snowflake + config: + username: ${SNOW_USER} + password: ${SNOW_PASS} + account_id: + warehouse: COMPUTE_WH + role: accountadmin + database_pattern: + allow: + - "long_tail_companions" + schema_pattern: + deny: + - information_schema + profiling: + enabled: False + domain: + "urn:li:domain:6289fccc-4af2-4cbb-96ed-051e7d1de93c": + allow: + - "long_tail_companions.analytics.*" + "urn:li:domain:07155b15-cee6-4fda-b1c1-5a19a6b74c3a": + allow: + - "long_tail_companions.ecommerce.*" +``` + + ## Searching by Domain @@ -137,4 +214,4 @@ Click [here](https://www.loom.com/share/72b3bcc2729b4df0982fa63ae3a8cb21) to see ## Feedback / Questions / Concerns -We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack! \ No newline at end of file +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack! diff --git a/docs/features.md b/docs/features.md index 8b8c3f10a9f0eb..a06789f28f82bd 100644 --- a/docs/features.md +++ b/docs/features.md @@ -4,142 +4,115 @@ title: "Features" # DataHub Features Overview -DataHub is a modern data catalog built to enable end-to-end data discovery, data observability, and data governance. This extensible metadata platform is built for developers to tame the complexity of their rapidly evolving data ecosystems, and for data practitioners to leverage the full value of data within their organization. +DataHub is a modern data catalog built to enable end-to-end data discovery, data observability, and data governance. This extensible metadata platform is built for developers to tame the complexity of their rapidly evolving data ecosystems and for data practitioners to leverage the total value of data within their organization. -Here’s an overview of DataHub’s current functionality. Curious about what’s to come? Check out our [roadmap](https://feature-requests.datahubproject.io/roadmap). +Here’s an overview of DataHub’s current functionality. Check out our [roadmap](https://feature-requests.datahubproject.io/roadmap) to see what's to come. -## End-to-end Search and Discovery +--- + +## Search and Discovery + +### **Search All Corners of Your Data Stack** + +DataHub's unified search experience surfaces results across databases, data lakes, BI platforms, ML feature stores, orchestration tools, and more. + +

+ +

+ +### **Trace End-to-End Lineage** -### Search for assets across databases, datalakes, BI platforms, ML feature stores, workflow orchestration, and more +Quickly understand the end-to-end journey of data by tracing lineage across platforms, datasets, ETL/ELT pipelines, charts, dashboards, and beyond. -Here’s an example of searching for assets related to the term `health`: we see results spanning Looker dashboards, BigQuery datasets, and DataHub Tags & Users, and ultimately navigate to the “DataHub Health” Looker dashboard overview ([view in demo site](https://demo.datahubproject.io/dashboard/urn:li:dashboard:(looker,dashboards.11)/Documentation?is_lineage_mode=false)) +

+ +

-![](./imgs/feature-search-across-all-entities.gif) +### **Understand the Impact of Breaking Changes on Downstream Dependencies** -### Easily understand the end-to-end journey of data by tracing lineage across platforms, datasets, pipelines, charts, and dashboards +Proactively identify which entities may be impacted by a breaking change using Impact Analysis. -Let’s dig into the dependency chain of the “DataHub Health” Looker dashboard. Using the lineage view, we can navigate all upstream dependencies of the Dashboard including Looker Charts, Snowflake and s3 Datasets, and Airflow Pipelines ([view in demo site](https://demo.datahubproject.io/dashboard/urn:li:dashboard:(looker,dashboards.11)/Documentation?is_lineage_mode=true)) +

+ +

-![](./imgs/feature-navigate-lineage-vis.gif) +### **View Metadata 360 at a Glance** -### Quickly gain context about related entities as you navigate the lineage graph +Combine *technical* and *logical* metadata to provide a 360º view of your data entities. -As you explore the relationships between entities, it’s easy to view documentation, usage stats, ownership, and more without leaving the lineage graph +Generate **Dataset Stats** to understand the shape & distribution of the data -![](./imgs/feature-view-entitiy-details-via-lineage-vis.gif) +

+ +

-### Gain confidence in the accuracy and relevance of datasets +Capture historical **Data Validation Outcomes** from tools like Great Expectations -DataHub provides dataset profiling and usage statistics for popular data warehousing platforms, making it easy for data practitioners to understand the shape of the data and how it has evolved over time. Query stats give context into how often (and by whom) the data is queried which can act as a strong signal of the trustworthiness of a dataset +

+ +

-![](./imgs/feature-table-usage-and-stats.gif) +Leverage DataHub's **Schema Version History** to track changes to the physical structure of data over time -## Robust Documentation and Tagging +

+ +

-### Capture and maintain institutional knowledge via API and/or the DataHub UI +--- -DataHub makes it easy to update and maintain documentation as definitions and use cases evolve. In addition to managing documentation via GMS, DataHub offers rich documentation and support for external links via the UI. +## Modern Data Governance -![](./imgs/feature-rich-documentation.gif) +### **Govern in Real Time** -### Create and define new tags via API and/or the DataHub UI +[The Actions Framework](./actions/README.md) powers the following real-time use cases: -Create and add tags to any type of entity within DataHub via the GraphQL API, or allow your end users to create and define new tags within the UI as use cases evolve over time +* **Notifications:** Generate organization-specific notifications when a change is made on DataHub. For example, send an email to the governance team when a "PII" tag is added to any data asset. +* **Workflow Integration:** Integrate DataHub into your organization's internal workflows. For example, create a Jira ticket when specific Tags or Terms are proposed on a Dataset. +* **Synchronization:** Sync changes made in DataHub into a 3rd party system. For example, reflect Tag additions in DataHub into Snowflake. +* **Auditing:** Audit who is making what changes on DataHub through time. -![](./imgs/feature-create-new-tag.gif) +

+ +

-### Browse and search specific tags to fast-track discovery across entities +### **Manage Entity Ownership** +Quickly and easily assign entity ownership to users and user groups. -Seamlessly browse entities associated with a tag or filter search results for a specific tag to find the entities that matter most +

+ +

-![](./imgs/feature-tag-browse.gif) +### **Govern with Tags, Glossary Terms, and Domains** +Empower data owners to govern their data entities with: -## Data Governance at your fingertips +1. **Tags:** Informal, loosely controlled labels that serve as a tool for search & discovery. No formal, central management. +2. **Glossary Terms:** A controlled vocabulary with optional hierarchy, commonly used to describe core business concepts and measurements. +3. **Domains:** Curated, top-level folders or categories, widely used in Data Mesh to organize entities by department (i.e., Finance, Marketing) or Data Products. -### Quickly assign asset ownership to users and/or user groups +

+ +

-![](./imgs/feature-add-owners.gif) +--- +## DataHub Administration -### Manage Fine-Grained Access Control with Policies +### **Create Users, Groups, & Access Policies** DataHub admins can create Policies to define who can perform what action against which resource(s). When you create a new Policy, you will be able to define the following: -* **Policy Type Platform** (top-level DataHub Platform privileges, i.e. managing users, groups, and policies) or Metadata (ability to manipulate ownership, tags, documentation, & more) -* **Resource Type** - Specify the type of resource, such as Datasets, Dashboards, Pipelines, etc. +* **Policy Type** - Platform (top-level DataHub Platform privileges, i.e., managing users, groups, and policies) or Metadata (ability to manipulate ownership, tags, documentation, and more) +* **Resource Type** - Specify the type of resources, such as Datasets, Dashboards, Pipelines, and beyond * **Privileges** - Choose the set of permissions, such as Edit Owners, Edit Documentation, Edit Links -* **Users and/or Groups** - Assign relevant Users and/or Groups; you can also assign the Policy to Resource Owners, regardless of which Group they belong to - -![](./imgs/feature-create-policy.gif) - -## Metadata quality & usage analytics - -Gain a deeper understanding of the health of metadata within DataHub and how end-users are interacting with the platform. The Analytics view provides a snapshot of volume of assets and percentage with assigned ownership, weekly active users, and most common searches & actions ([view in demo site](https://demo.datahubproject.io/analytics)). - -![](./imgs/feature-datahub-analytics.png) - -## DataHub is a Platform for Developers - -DataHub is an API- and stream-first platform, empowering developers to implement an instance tailored to their specific data stack. Our growing set of flexible integration models allow for push and pull metadata ingestion, as well as no-code metadata model extensions to quickly get up and running. - -### Dataset Sources -| Source | Status | -|---|:---:| -| Athena | Supported | -| BigQuery | Supported | -| Delta Lake | Planned | -| Druid | Supported | -| Elasticsearch | Supported | -| Hive | Supported | -| Hudi | Planned | -| Iceberg | Planned | -| Kafka Metadata | Supported | -| MongoDB | Supported | -| Microsoft SQL Server | Supported | -| MySQL | Supported | -| Oracle | Supported | -| PostgreSQL | Supported | -| Redshift | Supported | -| s3 | Supported | -| Snowflake | Supported | -| Spark/Databricks | Partially Supported | -| Trino FKA Presto | Supported | - -### BI Tools -| Source | Status | -|---|:---:| -| Business Glossary | Supported | -| Looker | Supported | -| Redash | Supported | -| Superset | Supported | -| Tableau | Planned | -| Grafana | Partially Supported | - -### ETL / ELT -| Source | Status | -|---|:---:| -| dbt | Supported | -| Glue | Supported | - -### Workflow Orchestration -| Source | Status | -|---|:---:| -| Airflow | Supported | -| Prefect | Planned | - -### Data Observability -| Source | Status | -|---|:---:| -| Great Expectations | Planned | - -### ML Platform -| Source | Status | -|---|:---:| -| Feast | Supported | -| Sagemaker | Supported | - -### Identity Management -| Source | Status | -|---|:---:| -| Azure AD | Supported | -| LDAP | Supported | -| Okta | Supported | \ No newline at end of file +* **Users and/or Groups** - Assign relevant Users and Groups; you can also assign the Policy to Resource Owners, regardless of which Group they belong + +

+ +

+ +### **Ingest Metadata from the UI** + +Create, configure, schedule, & execute batch metadata ingestion using the DataHub user interface. This makes getting metadata into DataHub easier by minimizing the overhead required to operate custom integration pipelines. + +

+ +

\ No newline at end of file diff --git a/docs/how/add-user-data.md b/docs/how/add-user-data.md index ba0027f5648bf2..ea76c97163ddda 100644 --- a/docs/how/add-user-data.md +++ b/docs/how/add-user-data.md @@ -1,10 +1,10 @@ -# Adding user in DataHub +# Adding user metadata in DataHub This guide shares how you can add user metadata in DataHub. Usually you would want to use one of our sources for ingesting user metadata. But if there is no connector for your use case then you would want to use this guide. :::note -This does not allow you to add new users for Authentication. If you want to add a new user in DataHub for Login please refer to [JaaS Authentication](./auth/jaas.md) +This does not allow you to add new users for Authentication. If you want to add a new user in DataHub for Login please refer to [Adding Users to DataHub](../authentication/guides/add-users.md) ::: diff --git a/docs/how/backup-datahub.md b/docs/how/backup-datahub.md index 3fccfd26349719..6a9d7287aaa45c 100644 --- a/docs/how/backup-datahub.md +++ b/docs/how/backup-datahub.md @@ -1,5 +1,11 @@ # Taking backup of DataHub +## Production + The recommended backup strategy is to periodically dump the database `datahub.metadata_aspect_v2` so it can be recreated from the dump which most managed DB services will support (e.g. AWS RDS). Then run [restore indices](./restore-indices.md) to recreate the indices. In order to back up Time Series Aspects (which power usage and dataset profiles), you'd have to do a backup of Elasticsearch, which is possible via AWS OpenSearch. Otherwise, you'd have to reingest dataset profiles from your sources in the event of a disaster scenario! + +## Quickstart + +To take a backup of your quickstart, take a look at this [document](../quickstart.md#backing-up-your-datahub-quickstart-experimental) on how to accomplish it. \ No newline at end of file diff --git a/docs/how/business-glossary-guide.md b/docs/how/business-glossary-guide.md new file mode 100644 index 00000000000000..3fb4f6245b592b --- /dev/null +++ b/docs/how/business-glossary-guide.md @@ -0,0 +1,90 @@ +# Business Glossary Guide + +## Introduction + +When working in complex data ecosystems, it is very useful to organize data assets using a shared vocabulary. The Business Glossary feature in DataHub helps you do this, by providing a framework for defining a standardized set of data concepts and then associating them with the physical assets that exist within your data ecosystem. + +Within this document, we'll introduce the core concepts comprising DataHub's Business Glossary feature and show you how to put it to work in your organization. + +### Terms & Term Groups + +A Business Glossary is comprised of two important primitives: Terms and Term Groups. + +- **Terms**: words or phrases with a specific business definition assigned to them. +- **Term Groups**: act like folders, containing Terms and even other Term Groups to allow for a nested structure. + +Both Terms and Term Groups allow you to add documentation and unique owners. + +For Glossary Terms, you are also able to establish relationships between different Terms in the **Related Terms** tab. Here you can create Contains and Inherits relationships. Finally, you can view all of the entities that have been tagged with a Term in the **Related Entities** tab. + +## Getting to your Glossary + +In order to view a Business Glossary, users must have the Platform Privilege called `Manage Glossaries` which can be granted by creating a new Platform [Policy](../authorization/policies.md). + +Once granted this privilege, you can access your Glossary by clicking the dropdown at the top of the page called **Govern** and then click **Glossary**: + +![](../imgs/glossary/glossary-button.png) + +You are now at the root of your Glossary and should see all Terms and Term Groups with no parents assigned to them. You should also notice a hierarchy navigator on the left where you can easily check out the structure of your Glossary! + +![](../imgs/glossary/root-glossary.png) + +## Creating a Term or Term Group + +There are two ways to create Terms and Term Groups through the UI. First, you can create directly from the Glossary home page by clicking the menu dots on the top right and selecting your desired option: + +![](../imgs/glossary/root-glossary-create.png) + +You can also create Terms or Term Groups directly from a Term Group's page. In order to do that you need to click the menu dots on the top right and select what you want: + +![](../imgs/glossary/create-from-node.png) + +Note that the modal that pops up will automatically set the current Term Group you are in as the **Parent**. You can easily change this by selecting the input and navigating through your Glossary to find your desired Term Group. In addition, you could start typing the name of a Term Group to see it appear by searching. You can also leave this input blank in order to create a Term or Term Group with no parent. + +![](../imgs/glossary/create-modal.png) + +## Editing a Term or Term Group + +In order to edit a Term or Term Group, you first need to go the page of the Term or Term group you want to edit. Then simply click the edit icon right next to the name to open up an inline editor. Change the text and it will save when you click outside or hit Enter. + +![](../imgs/glossary/edit-term.png) + +## Moving a Term or Term Group + +Once a Term or Term Group has been created, you can always move it to be under a different Term Group parent. In order to do this, click the menu dots on the top right of either entity and select **Move**. + +![](../imgs/glossary/move-term-button.png) + +This will open a modal where you can navigate through your Glossary to find your desired Term Group. + +![](../imgs/glossary/move-term-modal.png) + +## Deleting a Term or Term Group + +In order to delete a Term or Term Group, you need to go to the entity page of what you want to delete then click the menu dots on the top right. From here you can select **Delete** followed by confirming through a separate modal. **Note**: at the moment we only support deleting Term Groups that do not have any children. Until cascade deleting is supported, you will have to delete all children first, then delete the Term Group. + +![](../imgs/glossary/delete-button.png) + +## Adding a Term to an Entity + +Once you've defined your Glossary, you can begin attaching terms to data assets. To add a Glossary Term to an asset, go to the entity page of your asset and find the **Add Terms** button on the right sidebar. + +![](../imgs/glossary/add-term-to-entity.png) + +In the modal that pops up you can select the Term you care about in one of two ways: +- Search for the Term by name in the input +- Navigate through the Glossary dropdown that appears after clicking into the input + +![](../imgs/glossary/add-term-modal.png) + +## Demo + +Check out [our demo site](https://demo.datahubproject.io/glossary) to see an example Glossary and how it works! + +## Resources +- [Creating a Business Glossary and Putting it to use in DataHub](https://blog.datahubproject.io/creating-a-business-glossary-and-putting-it-to-use-in-datahub-43a088323c12) +- [Tags and Terms: Two Powerful DataHub Features, Used in Two Different Scenarios](https://medium.com/datahub-project/tags-and-terms-two-powerful-datahub-features-used-in-two-different-scenarios-b5b4791e892e) + +## Feedback / Questions / Concerns + +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack! diff --git a/docs/how/delete-metadata.md b/docs/how/delete-metadata.md index 62fdeb052a7276..8ac054193a3984 100644 --- a/docs/how/delete-metadata.md +++ b/docs/how/delete-metadata.md @@ -38,6 +38,7 @@ For now, this behaviour must be opted into by a prompt that will appear for you You can optionally add `-n` or `--dry-run` to execute a dry run before issuing the final delete command. You can optionally add `-f` or `--force` to skip confirmations +You can optionally add `--only-soft-deleted` flag to remove soft-deleted items only. :::note diff --git a/docs/how/kafka-config.md b/docs/how/kafka-config.md index 5f1ed11cd22a60..964aeb0cd88dde 100644 --- a/docs/how/kafka-config.md +++ b/docs/how/kafka-config.md @@ -3,11 +3,28 @@ title: "Configuring Kafka" hide_title: true --- -# How to configure Kafka? +# Configuring Kafka in DataHub + +DataHub requires Kafka to operate. Kafka is used as a durable log that can be used to store inbound +requests to update the Metadata Graph (Metadata Change Proposal), or as a change log detailing the updates +that have been made to the Metadata Graph (Metadata Change Log). + +## Environment Variables + +The following environment variables can be used to customize DataHub's connection to Kafka for the following DataHub components, +each of which requires a connection to Kafka: + +- `metadata-service` (datahub-gms container) +- (Advanced - if standalone consumers are deployed) `mce-consumer-job` (datahub-mce-consumer container) +- (Advanced - if standalone consumers are deployed) `mae-consumer-job` (datahub-mae-consumer container) +- (Advanced - if product analytics are enabled) datahub-frontend + +### Connection Configuration With the exception of `KAFKA_BOOTSTRAP_SERVER` and `KAFKA_SCHEMAREGISTRY_URL`, Kafka is configured via [spring-boot](https://spring.io/projects/spring-boot), specifically with [KafkaProperties](https://docs.spring.io/spring-boot/docs/current/api/org/springframework/boot/autoconfigure/kafka/KafkaProperties.html). See [Integration Properties](https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#integration-properties) prefixed with `spring.kafka`. Below is an example of how SASL/GSSAPI properties can be configured via environment variables: + ```bash export KAFKA_BOOTSTRAP_SERVER=broker:29092 export KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 @@ -16,6 +33,8 @@ export SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL=SASL_PLAINTEXT export SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG=com.sun.security.auth.module.Krb5LoginModule required principal='principal@REALM' useKeyTab=true storeKey=true keyTab='/keytab'; ``` +#### Example: Connecting using AWS IAM (MSK) + Here is another example of how SASL_SSL can be configured for AWS_MSK_IAM when connecting to MSK using IAM via environment variables ```bash SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL=SASL_SSL @@ -25,34 +44,60 @@ SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG=software.amazon.msk.auth.iam.IAMLoginMo SPRING_KAFKA_PROPERTIES_SASL_CLIENT_CALLBACK_HANDLER_CLASS=software.amazon.msk.auth.iam.IAMClientCallbackHandler ``` -These properties can be specified via `application.properties` or `application.yml` files, or as command line switches, or as environment variables. See Spring's [Externalized Configuration](https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config) to see how this works. +For more information about configuring these variables, check out Spring's [Externalized Configuration](https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config) to see how this works. +Also see [Kafka Connect Security](https://docs.confluent.io/current/connect/security.html) for more ways to connect. + + +### Topic Configuration + +By default, DataHub relies on the a set of Kafka topics to operate. By default, they have the following names: -See [Kafka Connect Security](https://docs.confluent.io/current/connect/security.html) for more ways to connect. +- **MetadataChangeProposal_v1** +- **FailedMetadataChangeProposal_v1** +- **MetadataChangeLog_Versioned_v1** +- **MetadataChangeLog_Timeseries_v1** +- **DataHubUsageEvent_v1**: User behavior tracking event for UI +6. (Deprecated) **MetadataChangeEvent_v4**: Metadata change proposal messages +7. (Deprecated) **MetadataAuditEvent_v4**: Metadata change log messages +8. (Deprecated) **FailedMetadataChangeEvent_v4**: Failed to process #1 event -DataHub components that connect to Kafka are currently: -- mce-consumer-job -- mae-consumer-job -- gms -- Various ingestion example apps +These topics are discussed at more length in [Metadata Events](../what/mxe.md). -## Configuring Topic Names +We've included environment variables to customize the name each of these topics, for cases where an organization has naming rules for your topics. -By default, ingestion relies upon the `MetadataChangeEvent_v4`, `MetadataAuditEvent_v4`, and `FailedMetadataChangeEvent` kafka topics by default for -[metadata events](../what/mxe.md). +### Metadata Service (datahub-gms) -We've included environment variables to customize the name each of these topics, if your company or organization has naming rules for your topics. +The following are environment variables you can use to configure topic names used in the Metadata Service container: -### datahub-gms -- `METADATA_CHANGE_EVENT_NAME`: The name of the metadata change event topic. -- `METADATA_AUDIT_EVENT_NAME`: The name of the metadata audit event topic. -- `FAILED_METADATA_CHANGE_EVENT_NAME`: The name of the failed metadata change event topic. +- `METADATA_CHANGE_PROPOSAL_TOPIC_NAME`: The name of the topic for Metadata Change Proposals emitted by the ingestion framework. +- `FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME`: The name of the topic for Metadata Change Proposals emitted when MCPs fail processing. +- `METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME`: The name of the topic for Metadata Change Logs that are produced for Versioned Aspects. +- `METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME`: The name of the topic for Metadata Change Logs that are produced for Timeseries Aspects. +- `METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME`: The name of the topic for Platform Events (high-level semantic events). +- `DATAHUB_USAGE_EVENT_NAME`: The name of the topic for product analytics events. +- (Deprecated) `METADATA_CHANGE_EVENT_NAME`: The name of the metadata change event topic. +- (Deprecated) `METADATA_AUDIT_EVENT_NAME`: The name of the metadata audit event topic. +- (Deprecated) `FAILED_METADATA_CHANGE_EVENT_NAME`: The name of the failed metadata change event topic. + +### MCE Consumer (datahub-mce-consumer) -### datahub-mce-consumer -- `KAFKA_MCE_TOPIC_NAME`: The name of the metadata change event topic. -- `KAFKA_FMCE_TOPIC_NAME`: The name of the failed metadata change event topic. +- `METADATA_CHANGE_PROPOSAL_TOPIC_NAME`: The name of the topic for Metadata Change Proposals emitted by the ingestion framework. +- `FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME`: The name of the topic for Metadata Change Proposals emitted when MCPs fail processing. +- (Deprecated) `METADATA_CHANGE_EVENT_NAME`: The name of the deprecated topic that an embedded MCE consumer will consume from. +- (Deprecated) `FAILED_METADATA_CHANGE_EVENT_NAME`: The name of the deprecated topic that failed MCEs will be written to. -### datahub-mae-consumer -- `KAFKA_TOPIC_NAME`: The name of the metadata audit event topic. +### MAE Consumer (datahub-mae-consumer) + +- `METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME`: The name of the topic for Metadata Change Logs that are produced for Versioned Aspects. +- `METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME`: The name of the topic for Metadata Change Logs that are produced for Timeseries Aspects. +- `METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME`: The name of the topic for Platform Events (high-level semantic events). +- `DATAHUB_USAGE_EVENT_NAME`: The name of the topic for product analytics events. +- (Deprecated) `METADATA_AUDIT_EVENT_NAME`: The name of the deprecated metadata audit event topic. + +### DataHub Frontend (datahub-frontend-react) + +- `DATAHUB_TRACKING_TOPIC`: The name of the topic used for storing DataHub usage events. +It should contain the same value as `DATAHUB_USAGE_EVENT_NAME` in the Metadata Service container. Please ensure that these environment variables are set consistently throughout your ecosystem. DataHub has a few different applications running which communicate with Kafka (see above). @@ -63,26 +108,60 @@ Kafka Consumers in Spring are configured using Kafka listeners. By default, cons We've included an environment variable to customize the consumer group id, if your company or organization has specific naming rules. ### datahub-mce-consumer and datahub-mae-consumer + - `KAFKA_CONSUMER_GROUP_ID`: The name of the kafka consumer's group id. -## How to apply configuration? -- For quickstart, add these environment variables to the corresponding application's docker.env -- For helm charts, add these environment variables as extraEnvs to the corresponding application's chart. -For example, +## Applying Configurations + +### Docker + +Simply add the above environment variables to the required `docker.env` files for the containers. These can +be found inside the `docker` folder of the repository. + + +### Helm + +On Helm, you'll need to configure these environment variables using the `extraEnvs` sections of the specific container's +configurations inside your `values.yaml` file. ``` -extraEnvs: - - name: METADATA_CHANGE_EVENT_NAME - value: "MetadataChangeEvent" - - name: METADATA_AUDIT_EVENT_NAME - value: "MetadataAuditEvent" - - name: FAILED_METADATA_CHANGE_EVENT_NAME - value: "FailedMetadataChangeEvent" - - name: KAFKA_CONSUMER_GROUP_ID - value: "my-apps-mae-consumer" +datahub-gms: + ... + extraEnvs: + - name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: "CustomMetadataChangeProposal_v1" + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: "CustomMetadataChangeLogVersioned_v1" + - name: FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: "CustomFailedMetadataChangeProposal_v1" + - name: KAFKA_CONSUMER_GROUP_ID + value: "my-apps-mae-consumer" + .... + +datahub-frontend: + ... + extraEnvs: + - name: DATAHUB_TRACKING_TOPIC + value: "MyCustomTrackingEvent" + +# If standalone consumers are enabled +datahub-mae-consumer; + extraEnvs: + - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME + value: "CustomMetadataChangeLogVersioned_v1" + .... + - name: METADATA_AUDIT_EVENT_NAME + value: "MetadataAuditEvent" +datahub-mce-consumer; + extraEnvs: + - name: METADATA_CHANGE_PROPOSAL_TOPIC_NAME + value: "CustomMetadataChangeLogVersioned_v1" + .... + - name: METADATA_CHANGE_EVENT_NAME + value: "MetadataChangeEvent" + .... ``` ## Other Components that use Kafka can be configured using environment variables: -- datahub-frontend - kafka-setup - schema-registry @@ -107,6 +186,7 @@ SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG=com.sun.security.auth.module.Krb5Log ## SSL ### Kafka + We are using the Spring Boot framework to start our apps, including setting up Kafka. You can [use environment variables to set system properties](https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config-relaxed-binding-from-environment-variables), including [Kafka properties](https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#integration-properties). diff --git a/docs/how/restore-indices.md b/docs/how/restore-indices.md index a5c0fd2b782d51..a9e2cd9799d19f 100644 --- a/docs/how/restore-indices.md +++ b/docs/how/restore-indices.md @@ -7,9 +7,18 @@ When a new version of the aspect gets ingested, GMS initiates an MAE event for t the search and graph indices. As such, we can fetch the latest version of each aspect in the local database and produce MAE events corresponding to the aspects to restore the search and graph indices. +## Quickstart + +If you're using the quickstart images, you can use the `datahub` cli to restore indices. + +``` +datahub docker quickstart --restore-indices +``` +See [this section](../quickstart.md#restoring-only-the-index-use-with-care) for more information. + ## Docker-compose -Run the following command from root to send MAE for each aspect in the Local DB. +If you are on a custom docker-compose deployment, run the following command (you need to checkout [the source repository](https://github.com/datahub-project/datahub)) from the root of the repo to send MAE for each aspect in the Local DB. ``` ./docker/datahub-upgrade/datahub-upgrade.sh -u RestoreIndices diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 6998da001554d5..c6b250c4d98316 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -12,6 +12,81 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Other notable Changes +## `v0.8.42` + +### Breaking Changes +- #5451 `GMS_HOST` and `GMS_PORT` environment variables deprecated in `v0.8.39` have been removed. Use `DATAHUB_GMS_HOST` and `DATAHUB_GMS_PORT` instead. +- #5478 DataHub CLI `delete` command when used with `--hard` option will delete soft-deleted entities which match the other filters given. +- #5471 Looker now populates `userEmail` in dashboard user usage stats. This version of looker connnector will not work with older version of **datahub-gms** if you have `extract_usage_history` looker config enabled. +- #5529 - `ANALYTICS_ENABLED` environment variable in **datahub-gms** is now deprecated. Use `DATAHUB_ANALYTICS_ENABLED` instead. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.41` + +### Breaking Changes +- The `should_overwrite` flag in `csv-enricher` has been replaced with `write_semantics` to match the format used for other sources. See the [documentation](https://datahubproject.io/docs/generated/ingestion/sources/csv/) for more details +- Closing an authorization hole in creating tags adding a Platform Privilege called `Create Tags` for creating tags. This is assigned to `datahub` root user, along +with default All Users policy. Notice: You may need to add this privilege (or `Manage Tags`) to existing users that need the ability to create tags on the platform. +- #5329 Below profiling config parameters are now supported in `BigQuery`: + - profiling.profile_if_updated_since_days (default=1) + - profiling.profile_table_size_limit (default=1GB) + - profiling.profile_table_row_limit (default=50000) + + Set above parameters to `null` if you want older behaviour. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.40` + +### Breaking Changes +- #5240 `lineage_client_project_id` in `bigquery` source is removed. Use `storage_project_id` instead. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.39` + +### Breaking Changes +- Refactored the `health` field of the `Dataset` GraphQL Type to be of type **list of HealthStatus** (was type **HealthStatus**). See [this PR](https://github.com/datahub-project/datahub/pull/5222/files) for more details. + +### Potential Downtime + +### Deprecations +- #5208 `GMS_HOST` and `GMS_PORT` environment variables being set in various containers are deprecated in favour of `DATAHUB_GMS_HOST` and `DATAHUB_GMS_PORT`. +- `KAFKA_TOPIC_NAME` environment variable in **datahub-mae-consumer** and **datahub-gms** is now deprecated. Use `METADATA_AUDIT_EVENT_NAME` instead. +- `KAFKA_MCE_TOPIC_NAME` environment variable in **datahub-mce-consumer** and **datahub-gms** is now deprecated. Use `METADATA_CHANGE_EVENT_NAME` instead. +- `KAFKA_FMCE_TOPIC_NAME` environment variable in **datahub-mce-consumer** and **datahub-gms** is now deprecated. Use `FAILED_METADATA_CHANGE_EVENT_NAME` instead. + + +### Other notable Changes +- #5132 Profile tables in `snowflake` source only if they have been updated since configured (default: `1`) number of day(s). Update the config `profiling.profile_if_updated_since_days` as per your profiling schedule or set it to `None` if you want older behaviour. + +## `v0.8.38` + +### Breaking Changes + +### Potential Downtime + +### Deprecations + +### Other notable Changes +- Create & Revoke Access Tokens via the UI +- Create and Manage new users via the UI +- Improvements to Business Glossary UI +- FIX - Do not require reindexing to migrate to using the UI business glossary + ## `v0.8.36` ### Breaking Changes diff --git a/docs/imgs/feature-add-owners.gif b/docs/imgs/feature-add-owners.gif deleted file mode 100644 index 245cf25d1d3199..00000000000000 Binary files a/docs/imgs/feature-add-owners.gif and /dev/null differ diff --git a/docs/imgs/feature-create-policy.gif b/docs/imgs/feature-create-policy.gif deleted file mode 100644 index 0cbfcdbd5b4ef9..00000000000000 Binary files a/docs/imgs/feature-create-policy.gif and /dev/null differ diff --git a/docs/imgs/feature-navigate-lineage-vis.gif b/docs/imgs/feature-navigate-lineage-vis.gif deleted file mode 100644 index c84303e2c0b837..00000000000000 Binary files a/docs/imgs/feature-navigate-lineage-vis.gif and /dev/null differ diff --git a/docs/imgs/feature-search-across-all-entities.gif b/docs/imgs/feature-search-across-all-entities.gif deleted file mode 100644 index eb7f7fcd360c06..00000000000000 Binary files a/docs/imgs/feature-search-across-all-entities.gif and /dev/null differ diff --git a/docs/imgs/feature-table-usage-and-stats.gif b/docs/imgs/feature-table-usage-and-stats.gif deleted file mode 100644 index 64ffe8a5d0d59d..00000000000000 Binary files a/docs/imgs/feature-table-usage-and-stats.gif and /dev/null differ diff --git a/docs/imgs/feature-validation-timeseries.png b/docs/imgs/feature-validation-timeseries.png new file mode 100644 index 00000000000000..28ce1daec5f32e Binary files /dev/null and b/docs/imgs/feature-validation-timeseries.png differ diff --git a/docs/imgs/generate-personal-access-token.png b/docs/imgs/generate-personal-access-token.png index 5b5cf888f9f900..fec45a89660e10 100644 Binary files a/docs/imgs/generate-personal-access-token.png and b/docs/imgs/generate-personal-access-token.png differ diff --git a/docs/imgs/glossary/add-term-modal.png b/docs/imgs/glossary/add-term-modal.png new file mode 100644 index 00000000000000..e32a9cb8d648c6 Binary files /dev/null and b/docs/imgs/glossary/add-term-modal.png differ diff --git a/docs/imgs/glossary/add-term-to-entity.png b/docs/imgs/glossary/add-term-to-entity.png new file mode 100644 index 00000000000000..7487a68c0d7559 Binary files /dev/null and b/docs/imgs/glossary/add-term-to-entity.png differ diff --git a/docs/imgs/glossary/create-from-node.png b/docs/imgs/glossary/create-from-node.png new file mode 100644 index 00000000000000..70638d083343c2 Binary files /dev/null and b/docs/imgs/glossary/create-from-node.png differ diff --git a/docs/imgs/glossary/create-modal.png b/docs/imgs/glossary/create-modal.png new file mode 100644 index 00000000000000..e84fb5a36e2d40 Binary files /dev/null and b/docs/imgs/glossary/create-modal.png differ diff --git a/docs/imgs/glossary/delete-button.png b/docs/imgs/glossary/delete-button.png new file mode 100644 index 00000000000000..3e0cc2a5b0a54a Binary files /dev/null and b/docs/imgs/glossary/delete-button.png differ diff --git a/docs/imgs/glossary/edit-term.png b/docs/imgs/glossary/edit-term.png new file mode 100644 index 00000000000000..62b0e425c8c4f3 Binary files /dev/null and b/docs/imgs/glossary/edit-term.png differ diff --git a/docs/imgs/glossary/glossary-button.png b/docs/imgs/glossary/glossary-button.png new file mode 100644 index 00000000000000..e4b8fd23935877 Binary files /dev/null and b/docs/imgs/glossary/glossary-button.png differ diff --git a/docs/imgs/glossary/move-term-button.png b/docs/imgs/glossary/move-term-button.png new file mode 100644 index 00000000000000..df03c820340eff Binary files /dev/null and b/docs/imgs/glossary/move-term-button.png differ diff --git a/docs/imgs/glossary/move-term-modal.png b/docs/imgs/glossary/move-term-modal.png new file mode 100644 index 00000000000000..0fda501911b2b0 Binary files /dev/null and b/docs/imgs/glossary/move-term-modal.png differ diff --git a/docs/imgs/glossary/root-glossary-create.png b/docs/imgs/glossary/root-glossary-create.png new file mode 100644 index 00000000000000..c91f397eb6213c Binary files /dev/null and b/docs/imgs/glossary/root-glossary-create.png differ diff --git a/docs/imgs/glossary/root-glossary.png b/docs/imgs/glossary/root-glossary.png new file mode 100644 index 00000000000000..1296c16b0dc3d1 Binary files /dev/null and b/docs/imgs/glossary/root-glossary.png differ diff --git a/docs/imgs/invite-users-button.png b/docs/imgs/invite-users-button.png new file mode 100644 index 00000000000000..a5d07a1c1e7e75 Binary files /dev/null and b/docs/imgs/invite-users-button.png differ diff --git a/docs/imgs/invite-users-popup.png b/docs/imgs/invite-users-popup.png new file mode 100644 index 00000000000000..621b1521eae752 Binary files /dev/null and b/docs/imgs/invite-users-popup.png differ diff --git a/docs/imgs/reset-credentials-screen.png b/docs/imgs/reset-credentials-screen.png new file mode 100644 index 00000000000000..4b680837b77ab1 Binary files /dev/null and b/docs/imgs/reset-credentials-screen.png differ diff --git a/docs/imgs/reset-user-password-button.png b/docs/imgs/reset-user-password-button.png new file mode 100644 index 00000000000000..5b1f3ee153d072 Binary files /dev/null and b/docs/imgs/reset-user-password-button.png differ diff --git a/docs/imgs/reset-user-password-popup.png b/docs/imgs/reset-user-password-popup.png new file mode 100644 index 00000000000000..ac2456dde4d4d3 Binary files /dev/null and b/docs/imgs/reset-user-password-popup.png differ diff --git a/docs/imgs/user-sign-up-screen.png b/docs/imgs/user-sign-up-screen.png new file mode 100644 index 00000000000000..88c2589203bd18 Binary files /dev/null and b/docs/imgs/user-sign-up-screen.png differ diff --git a/docs/quickstart.md b/docs/quickstart.md index 4730984fef0940..f8fc64e8e65f25 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -30,47 +30,39 @@ To deploy a new instance of DataHub, perform the following steps. ::: -4. To deploy DataHub, run the following CLI command from your terminal +4. To deploy a DataHub instance locally, run the following CLI command from your terminal ``` datahub docker quickstart ``` + This will deploy a DataHub instance using [docker-compose](https://docs.docker.com/compose/). + Upon completion of this step, you should be able to navigate to the DataHub UI at [http://localhost:9002](http://localhost:9002) in your browser. You can sign in using `datahub` as both the username and password. + 5. To ingest the sample metadata, run the following CLI command from your terminal ``` datahub docker ingest-sample-data ``` -That's it! To start pushing your company's metadata into DataHub, take a look at -the [Metadata Ingestion Framework](../metadata-ingestion/README.md). - -## Resetting DataHub - -To cleanse DataHub of all of it's state (e.g. before ingesting your own), you can use the CLI `nuke` command. - -``` -datahub docker nuke -``` - -## Updating DataHub locally +:::note -If you have been testing DataHub locally, a new version of DataHub got released and you want to try the new version then you can use below commands. +If you've enabled [Metadata Service Authentication](authentication/introducing-metadata-service-authentication.md), you'll need to provide a Personal Access Token +using the `--token ` parameter in the command. -``` -datahub docker nuke --keep-data -datahub docker quickstart -``` +::: -This will keep the data that you have ingested so far in DataHub and start a new quickstart with the latest version of DataHub. +That's it! Now feel free to play around with DataHub! -## Troubleshooting +## Troubleshooting Issues -### Command not found: datahub +
+Command not found: datahub + If running the datahub cli produces "command not found" errors inside your terminal, your system may be defaulting to an older version of Python. Try prefixing your `datahub` commands with `python3 -m`: @@ -86,9 +78,31 @@ if [ -d "$HOME/.local/bin" ] ; then PATH="$HOME/.local/bin:$PATH" fi ``` - -### Miscellaneous Docker issues - +
+ +
+ +Port Conflicts + + +By default the quickstart deploy will require the following ports to be free on your local machine: + - 3306 for MySQL + - 9200 for Elasticsearch + - 9092 for the Kafka broker + - 8081 for Schema Registry + - 2181 for ZooKeeper + - 9002 for the DataHub Web Application (datahub-frontend) + - 8080 for the DataHub Metadata Service (datahub-gms) + + In case the default ports conflict with software you are already running on your machine, you can override these ports by passing additional flags to the `datahub docker quickstart` command. + e.g. To override the MySQL port with 53306 (instead of the default 3306), you can say: `datahub docker quickstart --mysql-port 53306`. Use `datahub docker quickstart --help` to see all the supported options. + +
+ +
+ +Miscellaneous Docker issues + There can be misc issues with Docker, like conflicting containers and dangling volumes, that can often be resolved by pruning your Docker state with the following command. Note that this command removes all unused containers, networks, images (both dangling and unreferenced), and optionally, volumes. @@ -96,3 +110,112 @@ images (both dangling and unreferenced), and optionally, volumes. ``` docker system prune ``` + +
+ +
+ +Still stuck? + +Hop over to our [Slack community](https://slack.datahubproject.io) and ask for help in the [#troubleshoot](https://datahubspace.slack.com/archives/C029A3M079U) channel! +
+ +## Next Steps + +### Ingest Metadata + +To start pushing your company's metadata into DataHub, take a look at [UI-based Ingestion Guide](./ui-ingestion.md), or to run ingestion using the cli, look at the [Metadata Ingestion Guide](../metadata-ingestion/README.md). + +### Invite Users + +To add users to your deployment to share with your team check out our [Adding Users to DataHub](authentication/guides/add-users.md) + +### Enable Authentication + +To enable SSO, check out [Configuring OIDC Authentication](authentication/guides/sso/configure-oidc-react.md) or [Configuring JaaS Authentication](authentication/guides/jaas.md). + +To enable backend Authentication, check out [authentication in DataHub's backend](authentication/introducing-metadata-service-authentication.md#Configuring Metadata Service Authentication). + +### Move to Production + +We recommend deploying DataHub to production using Kubernetes. We provide helpful [Helm Charts](https://artifacthub.io/packages/helm/datahub/datahub) to help you quickly get up and running. Check out [Deploying DataHub to Kubernetes](./deploy/kubernetes.md) for a step-by-step walkthrough. + +## Other Common Operations + +### Stopping DataHub + +To stop DataHub's quickstart, you can issue the following command. + +``` +datahub docker quickstart --stop +``` + +### Resetting DataHub (a.k.a factory reset) + +To cleanse DataHub of all of its state (e.g. before ingesting your own), you can use the CLI `nuke` command. + +``` +datahub docker nuke +``` + +### Backing up your DataHub Quickstart (experimental) + +The quickstart image is not recommended for use as a production instance. See [Moving to production](#move-to-production) for recommendations on setting up your production cluster. However, in case you want to take a backup of your current quickstart state (e.g. you have a demo to your company coming up and you want to create a copy of the quickstart data so you can restore it at a future date), you can supply the `--backup` flag to quickstart. +``` +datahub docker quickstart --backup +``` +will take a backup of your MySQL image and write it by default to your `~/.datahub/quickstart/` directory as the file `backup.sql`. You can customize this by passing a `--backup-file` argument. +e.g. +``` +datahub docker quickstart --backup --backup-file /home/my_user/datahub_backups/quickstart_backup_2002_22_01.sql +``` +:::note + +Note that the Quickstart backup does not include any timeseries data (dataset statistics, profiles, etc.), so you will lose that information if you delete all your indexes and restore from this backup. + +::: + +### Restoring your DataHub Quickstart (experimental) +As you might imagine, these backups are restore-able. The following section describes a few different options you have to restore your backup. + +#### Restoring a backup (primary + index) [most common] +To restore a previous backup, run the following command: +``` +datahub docker quickstart --restore +``` +This command will pick up the `backup.sql` file located under `~/.datahub/quickstart` and restore your primary database as well as the elasticsearch indexes with it. + +To supply a specific backup file, use the `--restore-file` option. +``` +datahub docker quickstart --restore --restore-file /home/my_user/datahub_backups/quickstart_backup_2002_22_01.sql +``` + +#### Restoring only the index [to deal with index out of sync / corruption issues] +Another situation that can come up is the index can get corrupt, or be missing some update. In order to re-bootstrap the index from the primary store, you can run this command to sync the index with the primary store. +``` +datahub docker quickstart --restore-indices +``` + +#### Restoring a backup (primary but NO index) [rarely used] +Sometimes, you might want to just restore the state of your primary database (MySQL), but not re-index the data. To do this, you have to explicitly disable the restore-indices capability. + +``` +datahub docker quickstart --restore --no-restore-indices +``` + + +### Upgrading your local DataHub + +If you have been testing DataHub locally, a new version of DataHub got released and you want to try the new version then you can just issue the quickstart command again. It will pull down newer images and restart your instance without losing any data. + +``` +datahub docker quickstart +``` + +### Customization + +If you would like to customize the DataHub installation further, please download the [docker-compose.yaml](https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml) used by the cli tool, modify it as necessary and deploy DataHub by passing the downloaded docker-compose file: +``` +datahub docker quickstart --quickstart-compose-file +``` + diff --git a/docs/schema-history.md b/docs/schema-history.md index 40f1f2e947c734..00c56e1065cc77 100644 --- a/docs/schema-history.md +++ b/docs/schema-history.md @@ -17,7 +17,7 @@ the dataset schema are, and when those were reported. Here's an example from our ![](./imgs/schema-blame-latest-version.png) -If you click on an older version in the selector, you'll travel back in time and see what the schena looked like back then. Notice +If you click on an older version in the selector, you'll travel back in time and see what the schema looked like back then. Notice the changes here to the glossary terms for the `status` field, and to the descriptions for the `created_at` and `updated_at` fields. diff --git a/docs/slack.md b/docs/slack.md index 2efa44ecb2eb91..1cc0f5b5e01ce5 100644 --- a/docs/slack.md +++ b/docs/slack.md @@ -2,24 +2,43 @@ The DataHub Slack is a thriving and rapidly growing community - we can't wait for you to join us! -[Sign up here](https://slack.datahubproject.io) to join us on Slack and to subscribe to the DataHub Community newsletter. Already a member? [Log in here](https://slack.datahubproject.io). +_[Sign up here](https://slack.datahubproject.io) to join us on Slack and to subscribe to the DataHub Community newsletter. Already a member? [Log in here](https://slack.datahubproject.io)._ + +## Slack Guidelines + +In addition to our [Code of Conduct](CODE_OF_CONDUCT.md), we expect all Slack Community Members to respect the following guidelines: + +### Make use of threads + +Threads help us keep conversations contained and help us ensure we help you find a resolution and get you the support you need. + +Use threads when posting long messages and large blocks of code and/or stack trace - it is a MASSIVE help for us to keep track of the large volume of questions across our various support channels. + +### Do not post the same question across multiple channels + +If you're having a tough time getting the support you need (or aren't sure where to go!), please DM [@Maggie](https://datahubspace.slack.com/team/U0121TRV0FL) for support + +### Do not solicit members of our Slack + +The DataHub Community exists to collaborate with, learn from, and support one another. It is not a space to pitch your products or services directly to our members via public channels, private channels, or direct messages. + +We are excited to have a growing presence from vendors to help answer questions from Community Members as they may arise, but we have a strict 3-strike policy against solicitation: + +1. First occurrence: We'll give you a friendly but public reminder that the behavior is inappropriate according to our guidelines. +2. Second occurrence: We'll send you a DM warning that any additional violations will result in removal from the community. +3. Third occurrence: We'll delete or ban your account. + +We reserve the right to ban users without notice if they are clearly spamming our Community Members. ## Navigating DataHub Slack -Lets get you settled in -- +Lets get you settled in -- -- **Head over to [#introduce-yourself](https://datahubspace.slack.com/archives/C01PU1K6GDP) to, well, introduce yourself!** We’d love to learn more about you, what brings you here, and how we can support you -- **Not sure how to start?** You guessed it, check out [#getting-started](https://datahubspace.slack.com/archives/CV2KB471C) - we’ll point you in the right direction +- **Head over to [#introduce-yourself](https://datahubspace.slack.com/archives/C01PU1K6GDP) to, well, introduce yourself!** We'd love to learn more about you, what brings you here, and how we can support you +- **Not sure how to start?** You guessed it, check out [#getting-started](https://datahubspace.slack.com/archives/CV2KB471C) - we'll point you in the right direction - **Looking for general debugging help?** [#troubleshoot](https://datahubspace.slack.com/archives/C029A3M079U) is the place to go - **Need some live support from the Core DataHub Team?** Join us during our 2xWeek Office Hours via Zoom! Check out [#office-hours](https://datahubspace.slack.com/archives/C02AD211493) for more details -- **Looking for ways to contribute to the DataHub project?** Tell us all about it in [#contribute](https://datahubspace.slack.com/archives/C017W0NTZHR) -- **Have suggestions on how to make DataHub better?** We can’t wait to hear them in [#feature-requests](https://datahubspace.slack.com/archives/C02FWNS2F08) +- **Looking for ways to contribute to the DataHub project?** Tell us all about it in [#contribute](https://datahubspace.slack.com/archives/C017W0NTZHR) +- **Have suggestions on how to make DataHub better?** We can't wait to hear them in [#feature-requests](https://datahubspace.slack.com/archives/C02FWNS2F08) - **Excited to share your experience working with DataHub?** [#show-and-tell](https://datahubspace.slack.com/archives/C02FD9PLCA0) is the perfect channel for you -- **Need something else?** Reach out to [@Maggie](https://datahubspace.slack.com/team/U0121TRV0FL) - our Community Product Manager - -### Slack Best Practices - -Please keep in mind these Slack best-practices -- - -- **Make use of threads** for addressing specific questions/topics, and when posting large blocks of code - it helps us keep our channels and conversations organized -- **Avoid posting the same question in multiple channels** - if you’re having a tough time getting the support you need (or aren’t sure where to go!) please DM [@Maggie](https://datahubspace.slack.com/team/U0121TRV0FL) for support \ No newline at end of file +- **Need something else?** Reach out to [@Maggie](https://datahubspace.slack.com/team/U0121TRV0FL) - our Community Product Manager \ No newline at end of file diff --git a/docs/tags.md b/docs/tags.md index 0cc57d47abe199..380418f58d27f2 100644 --- a/docs/tags.md +++ b/docs/tags.md @@ -17,7 +17,7 @@ DataHub supports Tags, Glossary Terms, & Domains as distinct types of Metadata t ## Adding a Tag Users must have the Metadata Privilege called `Edit Tags` to add tags at the entity level, and the Privilege called `Edit Dataset Column Tags` to edit tags at the column level. These Privileges -can be granted by creating a new Metadata [Policy](./policies.md). +can be granted by creating a new Metadata [Policy](authorization/policies.md). To add a tag at the dataset or container level, simply navigate to the page for that entity and click on the "Add Tag" button. diff --git a/docs/townhall-history.md b/docs/townhall-history.md index 9bcdc68972802f..89389b3272bae8 100644 --- a/docs/townhall-history.md +++ b/docs/townhall-history.md @@ -2,6 +2,29 @@ A list of previous Town Halls, their planned schedule, and the recording of the meeting. +## 7/28/2022 +[Full YouTube video](https://youtu.be/Zrkf3Mzcvc4) + +## Agenda + +- Community Updates +- Project Updates +- Improvements to UI-Based Ingestion +- Sneak Preview - Bulk Edits via the UI +- Streamlined Metadata Ingestion +- DataHub 201: Metadata Enrichment + +## 6/30/2022 +[Full YouTube video](https://youtu.be/fAD53fEJ6m0) + +## Agenda + +- Community Updates +- Project Updates +- dbt Integration Updates +- CSV Ingestion Support +- DataHub 201 - Glossary Term Deep Dive + ## 5/26/2022 [Full YouTube video](https://youtu.be/taKb_zyowEE) diff --git a/docs/townhalls.md b/docs/townhalls.md index 67bb982cd73cc4..f9c3bb16150cd8 100644 --- a/docs/townhalls.md +++ b/docs/townhalls.md @@ -5,15 +5,9 @@ Currently it's held on the fourth Thursday of every month (with some exceptions It's the perfect venue to meet the team behind DataHub and other users, as well as to ask higher-level questions, such as roadmap and product direction. From time to time we also use the opportunity to showcase upcoming features. -## Coming up - May 26th, 2022 @ 9 AM US Pacific +## Meeting Invite & Agenda -## Meeting Invite - -You can join with this link https://zoom.datahubproject.io, or [RSVP](https://rsvp.datahubproject.io/) to get a calendar invite. - -## Agenda - -TBD! +You can join with this link https://zoom.datahubproject.io, or [RSVP](https://rsvp.datahubproject.io/) to get a calendar invite - this will always have the most up-to-date agenda for upcoming sessions. ## Past Meetings diff --git a/docs/ui-ingestion.md b/docs/ui-ingestion.md index 13a098d804a79a..235f1521c070ae 100644 --- a/docs/ui-ingestion.md +++ b/docs/ui-ingestion.md @@ -12,7 +12,7 @@ This document will describe the steps required to configure, schedule, and execu ### Prerequisites To view & manage UI-based metadata ingestion, you must have the `Manage Metadata Ingestion` & `Manage Secrets` - privileges assigned to your account. These can be granted by a [Platform Policy](./policies.md). + privileges assigned to your account. These can be granted by a [Platform Policy](authorization/policies.md). ![](./imgs/ingestion-privileges.png) @@ -112,7 +112,7 @@ _Referencing DataHub Secrets from a Recipe definition_ When the Ingestion Source with this Recipe executes, DataHub will attempt to 'resolve' Secrets found within the YAML. If a secret can be resolved, the reference is substituted for its decrypted value prior to execution. Secret values are not persisted to disk beyond execution time, and are never transmitted outside DataHub. -> **Attention**: Any DataHub users who have been granted the `Manage Secrets` [Platform Privilege](./policies.md) will be able to retrieve plaintext secret values using the GraphQL API. +> **Attention**: Any DataHub users who have been granted the `Manage Secrets` [Platform Privilege](authorization/policies.md) will be able to retrieve plaintext secret values using the GraphQL API. #### Step 3: Schedule Execution @@ -191,7 +191,7 @@ A variety of things can cause an ingestion run to fail. Common reasons for failu failures, metadata ingestion will fail. Ensure that the network where DataHub is deployed has access to the data source which you are trying to reach. -4. **Authentication**: If you've enabled [Metadata Service Authentication](https://datahubproject.io/docs/introducing-metadata-service-authentication/), you'll need to provide a Personal Access Token +4. **Authentication**: If you've enabled [Metadata Service Authentication](authentication/introducing-metadata-service-authentication.md), you'll need to provide a Personal Access Token in your Recipe Configuration. To so this, set the 'token' field of the sink configuration to contain a Personal Access Token: ![](./imgs/ingestion-with-token.png) diff --git a/docs/what/mxe.md b/docs/what/mxe.md index 669c5fa83de22c..0cbd84a5ad2d67 100644 --- a/docs/what/mxe.md +++ b/docs/what/mxe.md @@ -1,94 +1,430 @@ -# What is MXE (Metadata Events)? +# Metadata Events -The models defined in [snapshot](snapshot.md) and [delta](delta.md) are used to build the schema for several metadata Kafka events. As these events have the prefix `Metadata` and suffix `Event`, they’re collectively referred to as MXE. +DataHub makes use a few important Kafka events for operation. The most notable of these include -We also model MXEs using [PDL](https://linkedin.github.io/rest.li/pdl_schema) and rely on the [pegasus gradle plugin](https://linkedin.github.io/rest.li/setup/gradle#generateavroschema) to convert them into [AVSC](https://avro.apache.org/docs/current/spec.html). However, we also need to rename all the namespaces of the generated AVSC to avoid namespace clashes for projects that depend on both the PDL models and MXEs. +1. Metadata Change Proposal +2. Metadata Change Log (Versioned + Timeseries) +3. Platform Event -As the AVSC and PDL model schemas are 100% compatible, it’d be very easy to convert the in-memory representation from one to another using [Pegasus’ DataTranslator](https://linkedin.github.io/rest.li/avro_translation). +Each event is originally authored using [PDL]( https://linkedin.github.io/rest.li/DATA-Data-Schema-and-Templates), a modeling language developed by LinkedIn, and +then converted into their Avro equivalents, which are used when writing and reading the events to Kafka. -## Metadata Change Event (MCE) +In the document, we'll describe each of these events in detail - including notes about their structure & semantics. + +## Metadata Change Proposal (MCP) + +A Metadata Change Proposal represents a request to change to a specific [aspect](aspect.md) on an enterprise's Metadata +Graph. Each MCP provides a new value for a given aspect. For example, a single MCP can +be emitted to change ownership or documentation or domains or deprecation status for a data asset. + +### Emission + +MCPs may be emitted by clients of DataHub's low-level ingestion APIs (e.g. ingestion sources) +during the process of metadata ingestion. The DataHub Python API exposes an interface for +easily sending MCPs into DataHub. + +The default Kafka topic name for MCPs is `MetadataChangeProposal_v1`. + +### Consumption + +DataHub's storage layer actively listens for new Metadata Change Proposals, attempts +to apply the requested change to the Metadata Graph. + +### Schema + +| Name | Type | Description | Optional | +|---------------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| entityUrn | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | False | +| entityType | String | The type of the entity the new aspect is associated with. This corresponds to the entity name in the DataHub Entity Registry, for example 'dataset'. | False | +| entityKeyAspect | Object | The key struct of the entity that was changed. Only present if the Metadata Change Proposal contained the raw key struct. | True | +| changeType | String | The change type. CREATE, UPSERT and DELETE are currently supported. | False | +| aspectName | String | The entity aspect which was changed. | False | +| aspect | Object | The new aspect value. Null if the aspect was deleted. | True | +| aspect.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json`. | False | +| aspect.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| systemMetadata | Object | The new system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeProposal.pdl). + +### Examples -MCE is a "proposal" for a metadata change, as opposed to [MAE](#metadata-audit-event), which is conveying a committed change. -Consequently, only successfully accepted and processed MCEs will lead to the emission of a corresponding MAE. -A single MCE can contain both snapshot-oriented and delta-oriented metadata change proposal. The use case of this event is explained in [Metadata Ingestion](../architecture/metadata-ingestion.md). +An MCP representing a request to update the 'ownership' aspect for a particular Dataset: +```json +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\":[{\"type\":\"DATAOWNER\",\"owner\":\"urn:li:corpuser:datahub\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:datahub\",\"time\":1651516640488}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516640493, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + } +} ``` -namespace com.linkedin.mxe -import com.linkedin.avro2pegasus.events.KafkaAuditHeader -import com.linkedin.metadata.delta.Delta -import com.linkedin.metadata.snapshot.Snapshot +Note how the aspect payload is serialized as JSON inside the "value" field. The exact structure +of the aspect is determined by its PDL schema. (For example, the [ownership](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl) schema) + +## Metadata Change Log (MCL) + +A Metadata Change Log represents *any* change which has been made to the Metadata Graph. +Metadata Change Log events are emitted to Kafka immediately after writing the change +the durable storage. + +There are 2 flavors of Metadata Change Log: *versioned* and *timeseries*. These correspond to the type +of aspects which were updated for a given change. **Versioned** aspects are those +which represent the "latest" state of some attributes, for example the most recent owners of an asset +or its documentation. **Timeseries** aspects are those which represent events related to an asset +that occurred at a particular time, for example profiling of a Dataset. + +### Emission + +MCLs are emitted when *any* change is made to an entity on the DataHub Metadata Graph, this includes +writing to any aspect of an entity. + +Two distinct topics are maintained for Metadata Change Log. The default Kafka topic name for **versioned** aspects is `MetadataChangeLog_Versioned_v1` and for +**timeseries** aspects is `MetadataChangeLog_Timeseries_v1`. + +### Consumption + +DataHub ships with a Kafka Consumer Job (mae-consumer-job) which listens for MCLs and uses them to update DataHub's search and graph indices, +as well as to generate derived Platform Events (described below). + +In addition, the [Actions Framework](../actions/README.md) consumes Metadata Change Logs to power its [Metadata Change Log](../actions/events/metadata-change-log-event.md) event API. + +### Schema -/** - * Kafka event for proposing a metadata change for an entity - */ -record MetadataChangeEvent { +| Name | Type | Description | Optional | +|---------------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| entityUrn | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | False | +| entityType | String | The type of the entity the new aspect is associated with. This corresponds to the entity name in the DataHub Entity Registry, for example 'dataset'. | False | +| entityKeyAspect | Object | The key struct of the entity that was changed. Only present if the Metadata Change Proposal contained the raw key struct. | True | +| changeType | String | The change type. CREATE, UPSERT and DELETE are currently supported. | False | +| aspectName | String | The entity aspect which was changed. | False | +| aspect | Object | The new aspect value. Null if the aspect was deleted. | True | +| aspect.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json`. | False | +| aspect.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| previousAspectValue | Object | The previous aspect value. Null if the aspect did not exist previously. | True | +| previousAspectValue.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json` | False | +| previousAspectValue.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| systemMetadata | Object | The new system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | +| previousSystemMetadata | Object | The previous system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | +| created | Object | Audit stamp about who triggered the Metadata Change and when. | False | +| created.time | Number | The timestamp in milliseconds when the aspect change occurred. | False | +| created.actor | String | The URN of the actor (e.g. corpuser) that triggered the change. - /** Kafka audit header */ - auditHeader: optional KafkaAuditHeader +The PDL schema for can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeLog.pdl). - /** Snapshot of the proposed metadata change. Include only the aspects affected by the change in the snapshot. */ - proposedSnapshot: Snapshot +### Examples - /** Delta of the proposed metadata partial update */ - proposedDelta: optional Delta +An MCL corresponding to a change in the 'ownership' aspect for a particular Dataset: + +```json +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\":[{\"type\":\"DATAOWNER\",\"owner\":\"urn:li:corpuser:datahub\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:datahub\",\"time\":1651516640488}}", + "contentType": "application/json" + }, + "previousAspectValue": { + "value": "{\"owners\":[{\"owner\":\"urn:li:corpuser:jdoe\",\"type\":\"DATAOWNER\"},{\"owner\":\"urn:li:corpuser:datahub\",\"type\":\"DATAOWNER\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:jdoe\",\"time\":1581407189000}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516640493, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + }, + "previousSystemMetadata": { + "lastObserved": 1651516415088, + "runId": "file-2022_05_02-11_33_35", + "registryName": null, + "registryVersion": null, + "properties": null + }, + "created": { + "time": 1651516640490, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } } ``` -We’ll also generate a [dead letter queue](https://en.wikipedia.org/wiki/Dead_letter_queue) event, Failed Metadata Change Event (FMCE), for any rejected MCE. The event simply wraps the original MCE and an error message, which contains the reason for rejection. This event can be used for debugging any potential ingestion issues, as well as for re-playing any previous rejected proposal if ever needed. +Note how the aspect payload is serialized as JSON inside the "value" field. The exact structure +of the aspect is determined by its PDL schema. (For example, the [ownership](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl) schema) + + +## Platform Event (PE) + +A Platform Event represents an arbitrary business-logic event emitted by DataHub. Each +Platform Event has a `name` which determines its contents. + +### Types + +- **Entity Change Event** (entityChangeEvent): The most important Platform Event is named **Entity Change Event**, and represents a log of semantic changes +(tag addition, removal, deprecation change, etc) that have occurred on DataHub. It is used an important + component of the DataHub Actions Framework. + +All registered Platform Event types are declared inside the DataHub Entity Registry (`entity-registry.yml`). + +### Emission + +All Platform Events are generated by DataHub itself during normal operation. + +PEs are extremely dynamic - they can contain arbitrary payloads depending on the `name`. Thus, +can be emitted in a variety of circumstances. + +The default Kafka topic name for all Platform Events is `PlatformEvent_v1`. +### Consumption + +The [Actions Framework](../actions/README.md) consumes Platform Events to power its [Entity Change Event](../actions/events/entity-change-event.md) API. + +### Schema + +| Name | Type | Description | Optional | +|---------------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| header | Object | Header fields | False | +| header.timestampMillis | Long | The time at which the event was generated. | False | +| name | String | The name / type of the event. | False | +| payload | Object | The event itself. | False | +| payload.contentType | String | The serialization type of the event payload. The only supported value is `application/json`. | False | +| payload.value | String | The serialized payload. This is a JSON-serialized representing the payload document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | + + +The full PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/PlatformEvent.pdl). + +### Examples + +An example of an 'Entity Change Event' Platform Event that is emitted when a new owner is added to a Dataset: + +```json +{ + "header": { + "timestampMillis": 1655390732551 + }, + "name": "entityChangeEvent", + "payload": { + "value": "{\"entityUrn\":\"urn:li:dataset:abc\",\"entityType\":\"dataset\",\"category\":\"OWNER\",\"operation\":\"ADD\",\"modifier\":\"urn:li:corpuser:jdoe\",\"parameters\":{\"ownerUrn\":\"urn:li:corpuser:jdoe\",\"ownerType\":\"BUSINESS_OWNER\"},\"auditStamp\":{\"actor\":\"urn:li:corpuser:jdoe\",\"time\":1649953100653}}", + "contentType": "application/json" +} ``` -namespace com.linkedin.mxe -import com.linkedin.avro2pegasus.events.KafkaAuditHeader +Note how the actual payload for the event is serialized as JSON inside the 'payload' field. The exact +structure of the Platform Event is determined by its PDL schema. (For example, the [Entity Change Event](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/platform/event/v1/EntityChangeEvent.pdl) schema) + +## Failed Metadata Change Proposal (FMCP) + +When a Metadata Change Proposal cannot be processed successfully, the event is written to a [dead letter queue](https://en.wikipedia.org/wiki/Dead_letter_queue) +in an event called Failed Metadata Change Proposal (FMCP). -/** - * Kafka event for capturing a failure to process a specific MCE - */ -record FailedMetadataChangeEvent { +The event simply wraps the original Metadata Change Proposal and an error message, which contains the reason for rejection. +This event can be used for debugging any potential ingestion issues, as well as for re-playing any previous rejected proposal if necessary. - /** Kafka audit header */ - auditHeader: optional KafkaAuditHeader +### Emission - /** The event that failed to be processed */ - metadataChangeEvent: MetadataChangeEvent +FMCEs are emitted when MCEs cannot be successfully committed to DataHub's storage layer. - /** The error message or the stacktrace for the failure */ - error: string +The default Kafka topic name for FMCPs is `FailedMetadataChangeProposal_v1`. + +### Consumption + +No active consumers. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/FailedMetadataChangeProposal.pdl). + + +# Deprecated Events + +DataHub ships with a set of deprecated events, which were historically used for proposing and logging +changes to the Metadata Graph. + +Each event in this category was deprecated due to its inflexibility - namely the fact that +the schemas had to be updated when a new aspect was introduced. These events have since been replaced +by the more flexible events described above (Metadata Change Proposal, Metadata Change Log). + +It is not recommended to build dependencies on deprecated events. + +## Metadata Change Event (MCE) + +A Metadata Change Event represents a request to change multiple aspects for the same entity. +It leverages a deprecated concept of `Snapshot`, which is a strongly-typed list of aspects for the same +entity. + +A MCE is a "proposal" for a set of metadata changes, as opposed to [MAE](#metadata-audit-event), which is conveying a committed change. +Consequently, only successfully accepted and processed MCEs will lead to the emission of a corresponding MAE / MCLs. + +### Emission + +MCEs may be emitted by clients of DataHub's low-level ingestion APIs (e.g. ingestion sources) +during the process of metadata ingestion. + +The default Kafka topic name for MCEs is `MetadataChangeEvent_v4`. + +### Consumption + +DataHub's storage layer actively listens for new Metadata Change Events, attempts +to apply the requested changes to the Metadata Graph. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeEvent.pdl). + +### Examples + +An example of an MCE emitted to change the 'ownership' aspect for an Entity: + +```json +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + } } ``` ## Metadata Audit Event (MAE) -A Metadata Audit Event captures the change made to one or multiple metadata [aspects](aspect.md) associated with a particular [entity](entity.md), in the form of a metadata [snapshot](snapshot.md) before the change, and a metadata snapshot after the change. +A Metadata Audit Event captures changes made to one or multiple metadata [aspects](aspect.md) associated with a particular [entity](entity.md), in the form of a metadata [snapshot](snapshot.md) (deprecated) before the change, and a metadata snapshot after the change. Every source-of-truth for a particular metadata aspect is expected to emit a MAE whenever a change is committed to that aspect. By ensuring that, any listener of MAE will be able to construct a complete view of the latest state for all aspects. Furthermore, because each MAE contains the "after image", any mistake made in emitting the MAE can be easily mitigated by emitting a follow-up MAE with the correction. By the same token, the initial bootstrap problem for any newly added entity can also be solved by emitting a MAE containing all the latest metadata aspects associated with that entity. -``` -namespace com.linkedin.mxe - -import com.linkedin.avro2pegasus.events.KafkaAuditHeader -import com.linkedin.metadata.snapshot.Snapshot - -/** - * Kafka event for capturing update made to an entity's metadata. - */ -record MetadataAuditEvent { - - /** Kafka audit header */ - auditHeader: optional KafkaAuditHeader - - /** - * Snapshot of the metadata before the update. Set to null for newly created metadata. - * Only the metadata aspects affected by the update are included in the snapshot. - */ - oldSnapshot: optional Snapshot - - /** - * Snapshot of the metadata after the update. Only the metadata aspects affected by the - * update are included in the snapshot. - */ - newSnapshot: Snapshot +### Emission + +> Note: In recent versions of DataHub (mid 2022), MAEs are no longer actively emitted, and will soon be completely removed from DataHub. +> Use Metadata Change Log instead. + +MAEs are emitted once any metadata change has been successfully committed into DataHub's storage +layer. + +The default Kafka topic name for MAEs is `MetadataAuditEvent_v4`. + +### Consumption + +No active consumers. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataAuditEvent.pdl). + +### Examples + +An example of an MAE emitted representing a change made to the 'ownership' aspect for an Entity (owner removed): + +```json +{ + "oldSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "newSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + } } ``` + + +## Failed Metadata Change Event (FMCE) + +When a Metadata Change Event cannot be processed successfully, the event is written to a [dead letter queue](https://en.wikipedia.org/wiki/Dead_letter_queue) in an event called Failed Metadata Change Event (FMCE). + +The event simply wraps the original Metadata Change Event and an error message, which contains the reason for rejection. +This event can be used for debugging any potential ingestion issues, as well as for re-playing any previous rejected proposal if necessary. + +### Emission + +FMCEs are emitted when MCEs cannot be successfully committed to DataHub's storage layer. + +### Consumption + +No active consumers. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/FailedMetadataChangeEvent.pdl). + +The default Kafka topic name for FMCEs is `FailedMetadataChangeEvent_v4`. diff --git a/gradle.properties b/gradle.properties index 672b11f7b28b20..175a04383ee8c7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,7 +5,6 @@ org.gradle.caching=false # Increase gradle JVM memory to 3GB to allow tests to run locally org.gradle.jvmargs=-Xmx3000m - # Increase retries to 5 (from default of 3) and increase interval from 125ms to 1s. org.gradle.internal.repository.max.retries=5 org.gradle.internal.repository.max.tentatives=5 diff --git a/li-utils/build.gradle b/li-utils/build.gradle index 8124f883c832af..a0d50f56cb3811 100644 --- a/li-utils/build.gradle +++ b/li-utils/build.gradle @@ -4,7 +4,10 @@ apply plugin: 'pegasus' dependencies { compile spec.product.pegasus.data compile externalDependency.commonsLang - compile externalDependency.reflections + compile(externalDependency.reflections) { + exclude group: 'com.google.guava', module: 'guava' + } + compile externalDependency.guava compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/metadata-ingestion-examples/README.md b/metadata-ingestion-examples/README.md deleted file mode 100644 index d19438481f5625..00000000000000 --- a/metadata-ingestion-examples/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Metadata Ingestion - -**LEGACY** -This is a legacy module. The examples here are not actively maintained and may not work as described. Please see the `metadata-ingestion` module for more up-to-date uses. - -This directory contains example apps for ingesting data into DataHub. - -You are more than welcome to use these examples directly, or use them as a reference for you own jobs. - -See the READMEs of each example for more information on each. - -### Common themes - -All these examples ingest by firing MetadataChangeEvent Kafka events. They do not ingest directly into DataHub, though -this is possible. Instead, the mce-consumer-job should be running, listening for these events, and perform the ingestion -for us. - -### A note on languages - -We initially wrote these examples in Python (they still exist in `contrib`). The idea was that these were very small -example scripts, that should've been easy to use. However, upon reflection, not all developers are familiar with Python, -and the lack of types can hinder development. So the decision was made to port the examples to Java. - -You're more than welcome to extrapolate these examples into whatever languages you like. At LinkedIn, we primarily use -Java. - -### Ingestion at LinkedIn - -It is worth noting that we do not use any of these examples directly (in Java, Python, or anything else) at LinkedIn. We -have several different pipelines for ingesting data; it all depends on the source. - -- Some pipelines are based off other Kafka events, where we'll transform some existing Kafka event to a metadata event. - - For example, we get Kafka events hive changes. We make MCEs out of those hive events to ingest hive data. -- For others, we've directly instrumented existing pipelines / apps / jobs to also emit metadata events. -- For others still, we've created a series offline jobs to ingest data. - - For example, we have an Azkaban job to process our HDFS datasets. - -For some sources of data one of these example scripts may work fine. For others, it may make more sense to have some -custom logic, like the above list. Namely, all these examples today are one-off (they run, fire events, and then stop), -you may wish to build continuous ingestion pipelines instead. diff --git a/metadata-ingestion-examples/common/build.gradle b/metadata-ingestion-examples/common/build.gradle deleted file mode 100644 index d2d3637f6892cb..00000000000000 --- a/metadata-ingestion-examples/common/build.gradle +++ /dev/null @@ -1,29 +0,0 @@ -plugins { - id 'java' -} - -dependencies { - compile project(':metadata-dao-impl:kafka-producer') - - compile externalDependency.javaxInject - compile externalDependency.kafkaAvroSerde - compile externalDependency.lombok - compile externalDependency.springBeans - compile externalDependency.springBootAutoconfigure - compile externalDependency.springCore - compile externalDependency.springKafka - compile externalDependency.zookeeper - - annotationProcessor externalDependency.lombok - - runtime externalDependency.logbackClassic - - constraints { - implementation("org.apache.logging.log4j:log4j-core:2.17.0") { - because("previous versions are vulnerable to CVE-2021-45105") - } - implementation("org.apache.logging.log4j:log4j-api:2.17.0") { - because("previous versions are vulnerable to CVE-2021-45105") - } - } -} diff --git a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/KafkaConfig.java b/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/KafkaConfig.java deleted file mode 100644 index c5a1452978779f..00000000000000 --- a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/KafkaConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.linkedin.metadata.examples.configs; - -import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; -import io.confluent.kafka.serializers.KafkaAvroSerializer; -import java.util.Arrays; -import java.util.Map; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - - -@Configuration -public class KafkaConfig { - @Value("${KAFKA_BOOTSTRAP_SERVER:localhost:9092}") - private String kafkaBootstrapServers; - - @Value("${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081}") - private String kafkaSchemaRegistryUrl; - - @Bean(name = "kafkaProducer") - public Producer kafkaProducerFactory(KafkaProperties properties) { - KafkaProperties.Producer producerProps = properties.getProducer(); - - producerProps.setKeySerializer(StringSerializer.class); - producerProps.setValueSerializer(KafkaAvroSerializer.class); - - // KAFKA_BOOTSTRAP_SERVER has precedence over SPRING_KAFKA_BOOTSTRAP_SERVERS - if (kafkaBootstrapServers != null && kafkaBootstrapServers.length() > 0) { - producerProps.setBootstrapServers(Arrays.asList(kafkaBootstrapServers.split(","))); - } // else we rely on KafkaProperties which defaults to localhost:9092 - - Map props = properties.buildProducerProperties(); - props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaSchemaRegistryUrl); - - return new KafkaProducer<>(props); - } -} diff --git a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/SchemaRegistryConfig.java b/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/SchemaRegistryConfig.java deleted file mode 100644 index ceb1d05d6a7d9f..00000000000000 --- a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/SchemaRegistryConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.linkedin.metadata.examples.configs; - -import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; -import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - - -@Configuration -public class SchemaRegistryConfig { - @Value("${SCHEMAREGISTRY_URL:http://localhost:8081}") - private String schemaRegistryUrl; - - @Bean(name = "schemaRegistryClient") - public SchemaRegistryClient schemaRegistryFactory() { - return new CachedSchemaRegistryClient(schemaRegistryUrl, 512); - } -} diff --git a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/TopicConventionFactory.java b/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/TopicConventionFactory.java deleted file mode 100644 index eeb375355888eb..00000000000000 --- a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/TopicConventionFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.linkedin.metadata.examples.configs; - -import com.linkedin.mxe.TopicConvention; -import com.linkedin.mxe.TopicConventionImpl; -import com.linkedin.mxe.Topics; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - - -/** - * Creates a {@link TopicConvention} to generate kafka metadata event topic names. - * - *

This allows you to easily override Kafka topic names within your organization. - */ -@Configuration -public class TopicConventionFactory { - public static final String TOPIC_CONVENTION_BEAN = "metadataKafkaTopicConvention"; - - @Value("${METADATA_CHANGE_EVENT_NAME:" + Topics.METADATA_CHANGE_EVENT + "}") - private String metadataChangeEventName; - - @Value("${METADATA_AUDIT_EVENT_NAME:" + Topics.METADATA_AUDIT_EVENT + "}") - private String metadataAuditEventName; - - @Value("${FAILED_METADATA_CHANGE_EVENT_NAME:" + Topics.FAILED_METADATA_CHANGE_EVENT + "}") - private String failedMetadataChangeEventName; - - @Value("${METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + Topics.METADATA_CHANGE_PROPOSAL + "}") - private String metadataChangeProposalName; - - @Value("${METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME:" + Topics.METADATA_CHANGE_LOG_VERSIONED + "}") - private String metadataChangeLogVersionedTopicName; - - @Value("${METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME:" + Topics.METADATA_CHANGE_LOG_TIMESERIES + "}") - private String metadataChangeLogTimeseriesTopicName; - - @Value("${FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + Topics.FAILED_METADATA_CHANGE_PROPOSAL + "}") - private String failedMetadataChangeProposalName; - - @Value("${PLATFORM_EVENT_TOPIC_NAME:" + Topics.PLATFORM_EVENT + "}") - private String platformEventTopicName; - - @Bean(name = TOPIC_CONVENTION_BEAN) - protected TopicConvention createInstance() { - return new TopicConventionImpl(metadataChangeEventName, metadataAuditEventName, failedMetadataChangeEventName, - metadataChangeProposalName, metadataChangeLogVersionedTopicName, metadataChangeLogTimeseriesTopicName, - failedMetadataChangeProposalName, platformEventTopicName, - // TODO once we start rolling out v5 add support for changing the new event names. - TopicConventionImpl.DEFAULT_EVENT_PATTERN); - } -} diff --git a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/ZooKeeperConfig.java b/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/ZooKeeperConfig.java deleted file mode 100644 index 2923cd1a7ee647..00000000000000 --- a/metadata-ingestion-examples/common/src/main/java/com/linkedin/metadata/examples/configs/ZooKeeperConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.linkedin.metadata.examples.configs; - -import java.io.IOException; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooKeeper; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - - -@Configuration -public class ZooKeeperConfig { - @Value("${ZOOKEEPER:localhost:2181}") - private String zookeeper; - - @Value("${ZOOKEEPER_TIMEOUT_MILLIS:3000}") - private int timeoutMillis; - - @Bean(name = "zooKeeper") - public ZooKeeper zooKeeperFactory() throws IOException { - Watcher noopWatcher = event -> { - }; - - return new ZooKeeper(zookeeper, timeoutMillis, noopWatcher); - } -} diff --git a/metadata-ingestion-examples/kafka-etl/README.md b/metadata-ingestion-examples/kafka-etl/README.md deleted file mode 100644 index 2d5aa3d07bd255..00000000000000 --- a/metadata-ingestion-examples/kafka-etl/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Kafka ETL - -A small application which reads existing Kafka topics from ZooKeeper, retrieves their schema from the schema registry, -and then fires an MCE for each schema. - -## Running the Application - -First, ensure that services this depends on, like schema registry / zookeeper / mce-consumer-job / gms / etc, are all -running. - -This application can be run via gradle: - -``` -./gradlew :metadata-ingestion-examples:kafka-etl:bootRun -``` - -Or by building and running the jar: - -``` -./gradlew :metadata-ingestion-examples:kafka-etl:build - -java -jar metadata-ingestion-examples/kafka-etl/build/libs/kafka-etl.jar -``` - -### Environment Variables - -See the files under `src/main/java/com/linkedin/metadata/examples/kafka/config` for a list of customizable spring -environment variables. - -### Common pitfalls - -For events to be fired correctly, schemas must exist in the schema registry. If a topic was newly created, but no schema -has been registered for it yet, this application will fail to retrieve the schema for that topic. Check the output of -the application to see if this happens. If you see a message like - -``` -io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException: Subject not found.; error code: 40401 -``` - -Then the odds are good that you need to register the schema for this topic. \ No newline at end of file diff --git a/metadata-ingestion-examples/kafka-etl/build.gradle b/metadata-ingestion-examples/kafka-etl/build.gradle deleted file mode 100644 index 0ad4da77888a19..00000000000000 --- a/metadata-ingestion-examples/kafka-etl/build.gradle +++ /dev/null @@ -1,36 +0,0 @@ -plugins { - id 'org.springframework.boot' - id 'java' -} - -dependencies { - compile project(':metadata-utils') - compile project(':metadata-dao-impl:kafka-producer') - compile project(':metadata-events:mxe-schemas') - compile project(':metadata-ingestion-examples:common') - - compile externalDependency.javaxInject - compile externalDependency.kafkaAvroSerde - compile externalDependency.lombok - compile externalDependency.springBeans - compile externalDependency.springBootAutoconfigure - compile externalDependency.springCore - compile externalDependency.springKafka - - annotationProcessor externalDependency.lombok - - runtime externalDependency.logbackClassic - - constraints { - implementation("org.apache.logging.log4j:log4j-core:2.17.0") { - because("previous versions are vulnerable to CVE-2021-45105") - } - implementation("org.apache.logging.log4j:log4j-api:2.17.0") { - because("previous versions are vulnerable to CVE-2021-45105") - } - } -} - -bootJar { - mainClassName = 'com.linkedin.metadata.examples.kafka.KafkaEtlApplication' -} diff --git a/metadata-ingestion-examples/kafka-etl/src/main/java/com/linkedin/metadata/examples/kafka/KafkaEtl.java b/metadata-ingestion-examples/kafka-etl/src/main/java/com/linkedin/metadata/examples/kafka/KafkaEtl.java deleted file mode 100644 index eb5752052c53ad..00000000000000 --- a/metadata-ingestion-examples/kafka-etl/src/main/java/com/linkedin/metadata/examples/kafka/KafkaEtl.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.linkedin.metadata.examples.kafka; - -import com.linkedin.common.AuditStamp; -import com.linkedin.common.FabricType; -import com.linkedin.common.urn.CorpuserUrn; -import com.linkedin.common.urn.DataPlatformUrn; -import com.linkedin.common.urn.DatasetUrn; -import com.linkedin.metadata.aspect.DatasetAspect; -import com.linkedin.metadata.dao.producer.KafkaMetadataEventProducer; -import com.linkedin.metadata.examples.configs.TopicConventionFactory; -import com.linkedin.metadata.snapshot.DatasetSnapshot; -import com.linkedin.mxe.MetadataChangeEvent; -import com.linkedin.mxe.TopicConvention; -import com.linkedin.schema.KafkaSchema; -import com.linkedin.schema.SchemaField; -import com.linkedin.schema.SchemaFieldArray; -import com.linkedin.schema.SchemaFieldDataType; -import com.linkedin.schema.SchemaMetadata; -import com.linkedin.schema.StringType; -import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; -import java.util.List; -import javax.inject.Inject; -import javax.inject.Named; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.clients.producer.Producer; -import org.apache.zookeeper.ZooKeeper; -import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; - -/** - * Gathers Kafka topics from the local zookeeper instance and schemas from the schema registry, and then fires - * MetadataChangeEvents for their schemas. - * - *

This should cause DataHub to be populated with this information, assuming it and the mce-consumer-job are running - * locally. - * - *

Can be run with {@code ./gradlew :metadata-ingestion-examples:java:kafka-etl:bootRun}. - */ -@Slf4j -@Component -public final class KafkaEtl implements CommandLineRunner { - private static final DataPlatformUrn KAFKA_URN = new DataPlatformUrn("kafka"); - - @Inject - @Named("kafkaProducer") - private Producer _producer; - - @Inject - @Named(TopicConventionFactory.TOPIC_CONVENTION_BEAN) - private TopicConvention _topicConvention; - - @Inject - @Named("zooKeeper") - private ZooKeeper _zooKeeper; - - @Inject - @Named("schemaRegistryClient") - private SchemaRegistryClient _schemaRegistryClient; - - private SchemaMetadata buildDatasetSchema(String datasetName, String schema, int schemaVersion) { - final AuditStamp auditStamp = new AuditStamp(); - auditStamp.setTime(System.currentTimeMillis()); - auditStamp.setActor(new CorpuserUrn(System.getenv("USER"))); - final SchemaMetadata.PlatformSchema platformSchema = new SchemaMetadata.PlatformSchema(); - platformSchema.setKafkaSchema(new KafkaSchema().setDocumentSchema(schema)); - return new SchemaMetadata().setSchemaName(datasetName) - .setPlatform(KAFKA_URN) - .setCreated(auditStamp) - .setLastModified(auditStamp) - .setVersion(schemaVersion) - .setHash("") - .setPlatformSchema(platformSchema) - .setFields(new SchemaFieldArray(new SchemaField().setFieldPath("") - .setDescription("") - .setNativeDataType("string") - .setType(new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType()))))); - } - - private void produceKafkaDatasetMce(SchemaMetadata schemaMetadata) { - MetadataChangeEvent.class.getClassLoader().getResource("avro/com/linkedin/mxe/MetadataChangeEvent.avsc"); - - // Kafka topics are considered datasets in the current DataHub metadata ecosystem. - final KafkaMetadataEventProducer eventProducer = - new KafkaMetadataEventProducer<>(DatasetSnapshot.class, DatasetAspect.class, _producer, _topicConvention); - eventProducer.produceSnapshotBasedMetadataChangeEvent( - new DatasetUrn(KAFKA_URN, schemaMetadata.getSchemaName(), FabricType.PROD), schemaMetadata); - _producer.flush(); - } - - @Override - public void run(String... args) throws Exception { - log.info("Starting up"); - - final List topics = _zooKeeper.getChildren("/brokers/topics", false); - for (String datasetName : topics) { - if (datasetName.startsWith("_")) { - continue; - } - - final String topic = datasetName + "-value"; - io.confluent.kafka.schemaregistry.client.SchemaMetadata schemaMetadata; - try { - schemaMetadata = _schemaRegistryClient.getLatestSchemaMetadata(topic); - } catch (Throwable t) { - log.error("Failed to get schema for topic " + datasetName, t); - log.error("Common failure: does this event schema exist in the schema registry?"); - continue; - } - - if (schemaMetadata == null) { - log.warn(String.format("Skipping topic without schema: %s", topic)); - continue; - } - log.trace(topic); - - produceKafkaDatasetMce(buildDatasetSchema(datasetName, schemaMetadata.getSchema(), schemaMetadata.getVersion())); - log.info("Successfully fired MCE for " + datasetName); - } - } -} diff --git a/metadata-ingestion-examples/kafka-etl/src/main/java/com/linkedin/metadata/examples/kafka/KafkaEtlApplication.java b/metadata-ingestion-examples/kafka-etl/src/main/java/com/linkedin/metadata/examples/kafka/KafkaEtlApplication.java deleted file mode 100644 index 4eec1a5afb8e79..00000000000000 --- a/metadata-ingestion-examples/kafka-etl/src/main/java/com/linkedin/metadata/examples/kafka/KafkaEtlApplication.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.linkedin.metadata.examples.kafka; - -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration; -import org.springframework.boot.builder.SpringApplicationBuilder; - - -@SuppressWarnings("checkstyle:HideUtilityClassConstructor") -@SpringBootApplication(exclude = {ElasticsearchRestClientAutoConfiguration.class}, scanBasePackages = { - "com.linkedin.metadata.examples.configs", "com.linkedin.metadata.examples.kafka"}) -public class KafkaEtlApplication { - public static void main(String[] args) { - new SpringApplicationBuilder(KafkaEtlApplication.class).web(WebApplicationType.NONE).run(args); - } -} diff --git a/metadata-ingestion-examples/kafka-etl/src/main/resources/logback.xml b/metadata-ingestion-examples/kafka-etl/src/main/resources/logback.xml deleted file mode 100644 index 2c389931ec525f..00000000000000 --- a/metadata-ingestion-examples/kafka-etl/src/main/resources/logback.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - ${LOG_DIR}/kafka-etl-java.log - true - - %d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n - - - ${LOG_DIR}/kafka-etl.%i.log - 1 - 3 - - - 100MB - - - - - - - - - - - - - - - - - - diff --git a/metadata-ingestion-examples/mce-cli/README.md b/metadata-ingestion-examples/mce-cli/README.md deleted file mode 100644 index de524ad19f9672..00000000000000 --- a/metadata-ingestion-examples/mce-cli/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# MCE CLI - -A small application which can produce or consume [MetadataChangeEvents](../../docs/what/mxe.md). - -## Running the Application - -First, ensure that services this depends on, like schema registry / zookeeper / mce-consumer-job / gms / etc, are all -running. - -This application can be run via gradle: - -``` -./gradlew :metadata-ingestion-examples:mce-cli:bootRun -``` - -Or by building and running the jar: - -``` -./gradlew :metadata-ingestion-examples:mce-cli:build - -java -jar metadata-ingestion-examples/mce-cli/build/libs/mce-cli.jar -``` - -### Consuming Events - -Consuming MCEs may be useful to help debug other applications that are meant to produce them. You can easily see what -MCEs are being produced (or not) at a glance. - -``` -./gradlew :metadata-ingestion-examples:mce-cli:bootRun - -# Alternatives -./gradlew :metadata-ingestion-examples:mce-cli:bootRun --args='consume' -java -jar metadata-ingestion-examples/mce-cli/build/libs/mce-cli.jar -java -jar metadata-ingestion-examples/mce-cli/build/libs/mce-cli.jar consume -``` - -### Producing Events - -Producing events can be useful to help debug the MCE pipeline, or just to help make some fake data (ideally, don't do -this on your production stack!). - -``` -./gradlew :metadata-ingestion-examples:mce-cli:bootRun --args='-m produce my-file.json' - -# Alternatively -java -jar metadata-ingestion-examples/mce-cli/build/libs/mce-cli.jar -m produce my-file.json -``` - -Where `my-file.json` is some file that contains a -[MetadataChangEvents](./src/main/pegasus/com/linkedin/metadata/examples/cli/MetadataChangeEvents.pdl) JSON object. - -### Producing the Example Events with Docker - -We have some example events in the `example-bootstrap.json` file, which can be invoked via the above example. \ No newline at end of file diff --git a/metadata-ingestion-examples/mce-cli/build.gradle b/metadata-ingestion-examples/mce-cli/build.gradle deleted file mode 100644 index 5d887ae7997753..00000000000000 --- a/metadata-ingestion-examples/mce-cli/build.gradle +++ /dev/null @@ -1,43 +0,0 @@ -plugins { - id 'org.springframework.boot' - id 'java' - id 'pegasus' -} - -dependencies { - compile project(':metadata-utils') - compile project(':metadata-dao-impl:kafka-producer') - compile project(':metadata-events:mxe-schemas') - compile project(':metadata-ingestion-examples:common') - - dataModel project(':metadata-models') - - compile spec.product.pegasus.restliServer - compile externalDependency.javaxInject - compile externalDependency.kafkaAvroSerde - compile externalDependency.lombok - compile externalDependency.picocli - compile externalDependency.springBeans - compile externalDependency.springBootAutoconfigure - compile externalDependency.springCore - compile externalDependency.springKafka - - runtime externalDependency.logbackClassic - - annotationProcessor externalDependency.lombok - annotationProcessor externalDependency.picocli - - constraints { - implementation("org.apache.logging.log4j:log4j-core:2.17.0") { - because("previous versions are vulnerable to CVE-2021-45105") - } - implementation("org.apache.logging.log4j:log4j-api:2.17.0") { - because("previous versions are vulnerable to CVE-2021-45105") - } - } - -} - -bootJar { - mainClassName = 'com.linkedin.metadata.examples.cli.MceCliApplication' -} \ No newline at end of file diff --git a/metadata-ingestion-examples/mce-cli/example-bootstrap.json b/metadata-ingestion-examples/mce-cli/example-bootstrap.json deleted file mode 100644 index c2d86f88bcabf1..00000000000000 --- a/metadata-ingestion-examples/mce-cli/example-bootstrap.json +++ /dev/null @@ -1,602 +0,0 @@ -{ - "events": [ - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.CorpUserSnapshot": { - "urn": "urn:li:corpuser:datahub", - "aspects": [ - { - "com.linkedin.identity.CorpUserInfo": { - "active": true, - "displayName": "Data Hub", - "fullName": "Data Hub", - "email": "datahub@linkedin.com", - "title": "CEO" - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.CorpUserSnapshot": { - "urn": "urn:li:corpuser:jdoe", - "aspects": [ - { - "com.linkedin.identity.CorpUserInfo": { - "active": true, - "displayName": "John Doe", - "fullName": "John Doe", - "email": "jdoe@linkedin.com", - "title": "Software Engineer" - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", - "aspects": [ - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - }, - { - "com.linkedin.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - ] - } - }, - { - "com.linkedin.schema.SchemaMetadata": { - "schemaName": "SampleKafkaSchema", - "platform": "urn:li:dataPlatform:kafka", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "description": "Foo field description", - "nativeDataType": "string", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "glossaryTerms": { - "terms": [{ - "urn": "urn:li:glossaryTerm:instruments.FinancialInstrument_v1" - }], - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - }, - { - "fieldPath": "field_bar", - "description": "Bar field description", - "nativeDataType": "boolean", - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - } - } - ] - } - }, - { - "com.linkedin.common.GlossaryTerms": { - "terms": [{ - "urn": "urn:li:glossaryTerm:instruments.FinancialInstrument_v1" - }], - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", - "aspects": [ - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - }, - { - "com.linkedin.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - ] - } - }, - { - "com.linkedin.schema.SchemaMetadata": { - "schemaName": "SampleHdfsSchema", - "platform": "urn:li:dataPlatform:hdfs", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "description": "Foo field description", - "nativeDataType": "string", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - } - }, - { - "fieldPath": "field_bar", - "description": "Bar field description", - "nativeDataType": "boolean", - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - } - } - ] - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "aspects": [ - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - }, - { - "com.linkedin.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - ] - } - }, - { - "com.linkedin.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "description": "Foo field description", - "nativeDataType": "string", - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - } - }, - { - "fieldPath": "field_bar", - "description": "Bar field description", - "nativeDataType": "boolean", - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - } - } - ] - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.DataProcessSnapshot": { - "urn": "urn:li:dataProcess:(sqoop,DEMO,PROD)", - "aspects": [ - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub" - } - } - }, - { - "com.linkedin.dataprocess.DataProcessInfo": { - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:cassandra,barEarth,DEV)", - "urn:li:dataset:(urn:li:dataPlatform:cassandra,barMars,DEV)" - ], - "outputs": [ - "urn:li:dataset:(urn:li:dataPlatform:hbase,barSky,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:hbase,barOcean,PROD)" - ] - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlmodel:(urn:li:dataPlatform:science,scienceModel,PROD)", - "aspects": [ - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:jdoe" - } - } - }, - { - "com.linkedin.ml.metadata.MLModelProperties": { - "description": "A sample model for predicting some outcome.", - "date": 0, - "version": { - "versionTag": "1.5.3" - }, - "tags": [ - "science" - ] - } - }, - { - "com.linkedin.ml.metadata.TrainingData": { - "trainingData": [ - { - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,pageViewsHive,PROD)", - "motivation": "For science!", - "preProcessing": [ - "Aggregation" - ] - } - ] - } - }, - { - "com.linkedin.ml.metadata.EvaluationData": { - "evaluationData": [ - { - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,pageViewsHive,PROD)" - } - ] - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(Looker,1)", - "aspects": [ - { - "com.linkedin.chart.ChartInfo": { - "title": "Sample Looker Chart", - "description": "This chart contains sample data from Kafka", - "lastModified": { - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:datahub" - }, - "created": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)" - } - ], - "chartUrl": "https://www.looker.com", - "type": "BAR", - "access": "PUBLIC" - } - }, - { - "com.linkedin.chart.ChartQuery": { - "rawQuery": "SELECT * FROM SampleTable", - "type": "SQL" - } - }, - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "STAKEHOLDER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407589000, - "actor": "urn:li:corpuser:datahub" - } - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(Looker,0)", - "aspects": [ - { - "com.linkedin.dashboard.DashboardInfo": { - "title": "Sample Looker Dashboard", - "description": "This dashboard shows charts about user retention.", - "lastModified": { - "lastModified": { - "time": 1581407139000, - "actor": "urn:li:corpuser:datahub" - }, - "created": { - "time": 1581404189000, - "actor": "urn:li:corpuser:jdoe" - } - }, - "charts": [ "urn:li:chart:(Looker,1)" ], - "dashboardUrl": "https://www.looker.com", - "access": "PUBLIC" - } - }, - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" - }, - { - "owner": "urn:li:corpuser:jdoe", - "type": "STAKEHOLDER" - } - ], - "lastModified": { - "time": 1581407389000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:instruments.FinancialInstrument_v1", - "aspects": [ - { - "com.linkedin.glossary.GlossaryTermInfo": { - "definition": "written contract that gives rise to both a financial asset of one entity and a financial liability of another entity", - "parentNode": "urn:li:glossaryNode:instruments", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FinancialInstruments/FinancialInstruments/FinancialInstrument", - "customProperties": { - "FQDN": "common.instruments.FinancialInstrument" - } - } - }, - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - } - }, - { - "proposedSnapshot": { - "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot": { - "urn": "urn:li:glossaryNode:instruments", - "aspects": [ - { - "com.linkedin.glossary.GlossaryNodeInfo": { - "definition": "Financial Instruments" - } - }, - { - "com.linkedin.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - } - ], - "lastModified": { - "time": 1581407189000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] - } - } - } - ] -} \ No newline at end of file diff --git a/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/MceCli.java b/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/MceCli.java deleted file mode 100644 index 01c83c1fb5334e..00000000000000 --- a/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/MceCli.java +++ /dev/null @@ -1,127 +0,0 @@ -package com.linkedin.metadata.examples.cli; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.linkedin.data.DataMap; -import com.linkedin.data.schema.validation.RequiredMode; -import com.linkedin.data.schema.validation.UnrecognizedFieldMode; -import com.linkedin.data.schema.validation.ValidateDataAgainstSchema; -import com.linkedin.data.schema.validation.ValidationOptions; -import com.linkedin.data.schema.validation.ValidationResult; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.metadata.EventUtils; -import com.linkedin.mxe.MetadataChangeEvent; -import com.linkedin.mxe.Topics; -import com.linkedin.restli.common.ContentType; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.time.Duration; -import java.util.concurrent.ExecutionException; -import javax.annotation.Nonnull; -import javax.inject.Inject; -import javax.inject.Named; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; -import picocli.CommandLine; - - -@Slf4j -@Component -public class MceCli implements CommandLineRunner { - private enum Mode { - PRODUCE, CONSUME - } - - private static final class Args { - @CommandLine.Option(names = {"-m", "--mode"}, defaultValue = "CONSUME") - Mode mode; - - @CommandLine.Parameters( - paramLabel = "EVENT_FILE", - description = "MCE file; required if running 'producer' mode. See MetadataChangeEvents.pdl for schema.", - arity = "0..1" - ) - File eventFile; - } - - @Inject - @Named("kafkaProducer") - private Producer _producer; - - @Inject - @Named("kafkaEventConsumer") - private Consumer _consumer; - - private void consume() { - log.info("Consuming records."); - - _consumer.subscribe(ImmutableList.of(Topics.METADATA_CHANGE_EVENT)); - - while (true) { - final ConsumerRecords records = _consumer.poll(Duration.ofSeconds(1)); - - for (ConsumerRecord record : records) { - log.info(record.value().toString()); - } - } - } - - @VisibleForTesting - static MetadataChangeEvents readEventsFile(@Nonnull File eventsFile) throws IOException { - final DataMap dataMap = ContentType.JSON.getCodec().readMap(new FileInputStream(eventsFile)); - - final ValidationOptions options = new ValidationOptions(); - options.setRequiredMode(RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT); - options.setUnrecognizedFieldMode(UnrecognizedFieldMode.DISALLOW); - - final ValidationResult result = - ValidateDataAgainstSchema.validate(dataMap, DataTemplateUtil.getSchema(MetadataChangeEvents.class), options); - - if (!result.isValid()) { - throw new IllegalArgumentException( - String.format("Error parsing metadata events file %s: \n%s", eventsFile.toString(), - Joiner.on('\n').join(result.getMessages()))); - } - - return DataTemplateUtil.wrap(dataMap, MetadataChangeEvents.class); - } - - private void produce(@Nonnull File eventsFile) throws IOException, ExecutionException, InterruptedException { - final MetadataChangeEvents events = readEventsFile(eventsFile); - int record = 1; - for (MetadataChangeEvent mce : events.getEvents()) { - log.info("Producing record {} of {}", record++, events.getEvents().size()); - _producer.send(new ProducerRecord(Topics.METADATA_CHANGE_EVENT, EventUtils.pegasusToAvroMCE(mce))).get(); - log.info("Produced record."); - } - } - - @Override - public void run(String... cmdLineArgs) throws Exception { - final Args args = new Args(); - new CommandLine(args).setCaseInsensitiveEnumValuesAllowed(true).parseArgs(cmdLineArgs); - - switch (args.mode) { - case CONSUME: - consume(); - break; - case PRODUCE: - if (args.eventFile == null) { - throw new IllegalArgumentException("Event file is required when producing."); - } - produce(args.eventFile); - break; - default: - break; - } - } -} diff --git a/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/MceCliApplication.java b/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/MceCliApplication.java deleted file mode 100644 index 4a8eac0f40d74c..00000000000000 --- a/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/MceCliApplication.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.linkedin.metadata.examples.cli; - -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration; -import org.springframework.boot.builder.SpringApplicationBuilder; - - -@SuppressWarnings("checkstyle:HideUtilityClassConstructor") -@SpringBootApplication(exclude = {ElasticsearchRestClientAutoConfiguration.class}, scanBasePackages = { - "com.linkedin.metadata.examples.configs", "com.linkedin.metadata.examples.cli"}) -public class MceCliApplication { - public static void main(String[] args) { - new SpringApplicationBuilder(MceCliApplication.class).web(WebApplicationType.NONE).run(args); - } -} diff --git a/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/config/KafkaConsumerConfig.java b/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/config/KafkaConsumerConfig.java deleted file mode 100644 index 072b40dc9b7ad9..00000000000000 --- a/metadata-ingestion-examples/mce-cli/src/main/java/com/linkedin/metadata/examples/cli/config/KafkaConsumerConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.linkedin.metadata.examples.cli.config; - -import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; -import io.confluent.kafka.serializers.KafkaAvroDeserializer; -import java.util.Arrays; -import java.util.Map; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - - -@Configuration -public class KafkaConsumerConfig { - @Value("${KAFKA_BOOTSTRAP_SERVER:localhost:9092}") - private String kafkaBootstrapServers; - - @Value("${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081}") - private String kafkaSchemaRegistryUrl; - - @Bean(name = "kafkaEventConsumer") - public Consumer kafkaConsumerFactory(KafkaProperties properties) { - KafkaProperties.Consumer consumerProps = properties.getConsumer(); - - consumerProps.setKeyDeserializer(StringDeserializer.class); - consumerProps.setValueDeserializer(KafkaAvroDeserializer.class); - consumerProps.setGroupId("mce-cli"); - - // KAFKA_BOOTSTRAP_SERVER has precedence over SPRING_KAFKA_BOOTSTRAP_SERVERS - if (kafkaBootstrapServers != null && kafkaBootstrapServers.length() > 0) { - consumerProps.setBootstrapServers(Arrays.asList(kafkaBootstrapServers.split(","))); - } // else we rely on KafkaProperties which defaults to localhost:9092 - - Map props = properties.buildConsumerProperties(); - props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaSchemaRegistryUrl); - - return new KafkaConsumer<>(props); - } -} diff --git a/metadata-ingestion-examples/mce-cli/src/main/pegasus/com/linkedin/metadata/examples/cli/MetadataChangeEvents.pdl b/metadata-ingestion-examples/mce-cli/src/main/pegasus/com/linkedin/metadata/examples/cli/MetadataChangeEvents.pdl deleted file mode 100644 index 5ee3aa5bad737f..00000000000000 --- a/metadata-ingestion-examples/mce-cli/src/main/pegasus/com/linkedin/metadata/examples/cli/MetadataChangeEvents.pdl +++ /dev/null @@ -1,13 +0,0 @@ -namespace com.linkedin.metadata.examples.cli - -import com.linkedin.mxe.MetadataChangeEvent - -/** - * Schema definition for the format of the input file to the CLI for producing events. - */ -record MetadataChangeEvents { - /** - * Events to produce. - */ - events: array[MetadataChangeEvent] -} \ No newline at end of file diff --git a/metadata-ingestion-examples/mce-cli/src/main/resources/logback.xml b/metadata-ingestion-examples/mce-cli/src/main/resources/logback.xml deleted file mode 100644 index 164930daf0028e..00000000000000 --- a/metadata-ingestion-examples/mce-cli/src/main/resources/logback.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - ${LOG_DIR}/kafka-etl-java.log - true - - %d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n - - - ${LOG_DIR}/kafka-etl.%i.log - 1 - 3 - - - 100MB - - - - - - - - - - - - - - - - - - diff --git a/metadata-ingestion-examples/mce-cli/src/test/java/com/linkedin/metadata/examples/cli/TestExamples.java b/metadata-ingestion-examples/mce-cli/src/test/java/com/linkedin/metadata/examples/cli/TestExamples.java deleted file mode 100644 index 8d993aa8227d57..00000000000000 --- a/metadata-ingestion-examples/mce-cli/src/test/java/com/linkedin/metadata/examples/cli/TestExamples.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.linkedin.metadata.examples.cli; - -import com.linkedin.restli.common.ContentType; -import java.io.File; -import java.io.FileInputStream; -import org.testng.annotations.Test; - -import static org.testng.Assert.*; - - -/** - * Simple test to help us keep our example file up to date with the MCE schema definition, in the event we change - * schemas, or change the file without manually testing it (which we shouldn't do, but can happen by mistake). - */ -public class TestExamples { - private static final File EXAMPLE_FILE = new File("example-bootstrap.json"); - - @Test - public void examplesAreValidJson() throws Exception { - assertTrue(EXAMPLE_FILE.exists()); - // no exception = test passes - ContentType.JSON.getCodec().readMap(new FileInputStream(EXAMPLE_FILE)); - } - - @Test - public void examplesMatchSchemas() throws Exception { - // no exception = test passes - MceCli.readEventsFile(EXAMPLE_FILE); - } -} diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index c1e4e07b9712a7..e8b1b0839187d2 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -88,7 +88,7 @@ task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { } task cleanPythonCache(type: Exec) { - commandLine 'bash', '-x', '-c', + commandLine 'bash', '-c', "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete" } diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index 8ca4cad470f67e..6d26cf4498bdf9 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -1,4 +1,5 @@ import os +import pathlib import sys from typing import Dict, Set @@ -14,14 +15,11 @@ def get_long_description(): root = os.path.dirname(__file__) - with open(os.path.join(root, "README.md")) as f: - description = f.read() - - return description + return pathlib.Path(os.path.join(root, "README.md")).read_text() base_requirements = { - # Compatability. + # Compatibility. "dataclasses>=0.6; python_version < '3.7'", "typing_extensions>=3.10.0.2", "mypy_extensions>=0.4.3", diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py index c893ff61cb9ec3..f7b384c1d32390 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py @@ -1,3 +1,4 @@ +import contextlib import traceback from typing import Any, Iterable @@ -215,7 +216,7 @@ def datahub_on_success_callback(context, *args, **kwargs): for inlet in inlets: datajob.inlets.append(inlet.urn) - # We have to use _oulets because outlets is empty + # We have to use _outlets because outlets is empty for outlet in task._outlets: datajob.outlets.append(outlet.urn) @@ -389,13 +390,10 @@ def _patch_policy(settings): def _patch_datahub_policy(): - try: + with contextlib.suppress(ImportError): import airflow_local_settings _patch_policy(airflow_local_settings) - except ImportError: - pass - from airflow.models.dagbag import settings _patch_policy(settings) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/integration_test_dummy.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/integration_test_dummy.py index f4f53619168f89..10cf3ad0a608ae 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/integration_test_dummy.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/integration_test_dummy.py @@ -1,2 +1,2 @@ def test_dummy(): - assert True + pass diff --git a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_dummy.py b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_dummy.py index f4f53619168f89..10cf3ad0a608ae 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/unit/test_dummy.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/unit/test_dummy.py @@ -1,2 +1,2 @@ def test_dummy(): - assert True + pass diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md index 260b25fb0f082c..6cdc35a076fe9d 100644 --- a/metadata-ingestion/README.md +++ b/metadata-ingestion/README.md @@ -54,7 +54,7 @@ Make sure yaml plugin is installed for your editor: ::: Since `acryl-datahub` version `>=0.8.33.2`, the default sink is assumed to be a DataHub REST endpoint: -- Hosted at "http://localhost:8080" or the environment variable `${DATAHUB_GMS_HOST}` if present +- Hosted at "http://localhost:8080" or the environment variable `${DATAHUB_GMS_URL}` if present - With an empty auth token or the environment variable `${DATAHUB_GMS_TOKEN}` if present. Here's a simple recipe that pulls metadata from MSSQL (source) and puts it into the default sink (datahub rest). @@ -79,7 +79,7 @@ datahub ingest -c recipe.dhub.yaml or if you want to override the default endpoints, you can provide the environment variables as part of the command like below: ```shell -DATAHUB_GMS_HOST="https://my-datahub-server:8080" DATAHUB_GMS_TOKEN="my-datahub-token" datahub ingest -c recipe.dhub.yaml +DATAHUB_GMS_URL="https://my-datahub-server:8080" DATAHUB_GMS_TOKEN="my-datahub-token" datahub ingest -c recipe.dhub.yaml ``` A number of recipes are included in the [examples/recipes](./examples/recipes) directory. For full info and context on each source and sink, see the pages described in the [table of plugins](../docs/cli.md#installing-plugins). diff --git a/metadata-ingestion/archived/source_docs/dbt.md b/metadata-ingestion/archived/source_docs/dbt.md index d10dd0a85ee82b..d973d7d4f7a25c 100644 --- a/metadata-ingestion/archived/source_docs/dbt.md +++ b/metadata-ingestion/archived/source_docs/dbt.md @@ -89,6 +89,38 @@ dbt allows authors to define meta properties for datasets. Checkout this link to actions such as add a tag, term or owner. For example if a dbt model has a meta config ```"has_pii": True```, we can define an action that evaluates if the property is set to true and add, lets say, a ```pii``` tag. To leverage this feature we require users to define mappings as part of the recipe. Following is how mappings can be defined - + + +```yaml +meta_mapping: + business_owner: + match: ".*" + operation: "add_owner" + config: + owner_type: user + has_pii: + match: True + operation: "add_tag" + config: + tag: "has_pii_test" + int_property: + match: 1 + operation: "add_tag" + config: + tag: "int_meta_property" + double_property: + match: 2.5 + operation: "add_term" + config: + term: "double_meta_property" + data_governance.team_owner: + match: "Finance" + operation: "add_term" + config: + term: "Finance_test" +``` + + ```json "meta_mapping": { "business_owner": { @@ -118,6 +150,8 @@ To leverage this feature we require users to define mappings as part of the reci }, } ``` + + We support the below actions - 1. add_tag - Requires ```tag``` property in config. 2. add_term - Requires ```term``` property in config. @@ -135,7 +169,7 @@ This works similarly as the dbt meta mapping but for the query tags We support the below actions - 1. add_tag - Requires ```tag``` property in config. -The below example set as global tag the query tag `tag` key's value. +The below example set as global tag the query tag `tag` key's value. ```json "query_tag_mapping": { diff --git a/metadata-ingestion/archived/source_docs/elastic_search.md b/metadata-ingestion/archived/source_docs/elastic_search.md index 8e30e17dc34714..cbb43ee2837738 100644 --- a/metadata-ingestion/archived/source_docs/elastic_search.md +++ b/metadata-ingestion/archived/source_docs/elastic_search.md @@ -41,6 +41,9 @@ source: index_pattern: allow: [".*some_index_name_pattern*"] deny: [".*skip_index_name_pattern*"] + ingest_index_templates: False + index_template_pattern: + allow: [".*some_index_template_name_pattern*"] sink: # sink configs @@ -51,17 +54,20 @@ sink: Note that a `.` is used to denote nested fields in the YAML recipe. -| Field | Required | Default | Description | -| --------------------------- | -------- |--------------------|---------------------------------------------------------------| -| `host` | ✅ | `"localhost:9092"` | The elastic search host URI. | -| `username` | | None | The username credential. | -| `password` | | None | The password credential. | -| `url_prefix` | | "" | There are cases where an enterprise would have multiple elastic search clusters. One way for them to manage is to have a single endpoint for all the elastic search clusters and use url_prefix for routing requests to different clusters. | -| `env` | | `"PROD"` | Environment to use in namespace when constructing URNs. | -| `platform_instance` | | None | The Platform instance to use while constructing URNs. | -| `index_pattern.allow` | | | List of regex patterns for indexes to include in ingestion. | -| `index_pattern.deny` | | | List of regex patterns for indexes to exclude from ingestion. | -| `index_pattern.ignoreCase` | | `True` | Whether regex matching should ignore case or not | +| Field | Required | Default | Description | +|--------------------------------| -------- |--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `host` | ✅ | `"localhost:9092"` | The elastic search host URI. | +| `username` | | None | The username credential. | +| `password` | | None | The password credential. | +| `url_prefix` | | "" | There are cases where an enterprise would have multiple elastic search clusters. One way for them to manage is to have a single endpoint for all the elastic search clusters and use url_prefix for routing requests to different clusters. | +| `env` | | `"PROD"` | Environment to use in namespace when constructing URNs. | +| `platform_instance` | | None | The Platform instance to use while constructing URNs. | +| `index_pattern.allow` | | | List of regex patterns for indexes to include in ingestion. | +| `index_pattern.deny` | | | List of regex patterns for indexes to exclude from ingestion. | +| `index_pattern.ignoreCase` | | `True` | Whether regex matching should ignore case or not | +| `ingest_index_templates` | | `False` | Whether index templates should be ingested | +| `index_template_pattern.allow` | | | List of regex patterns for index templates to include in ingestion. | +| `index_template_pattern.deny` | | | List of regex patterns for index templates to exclude from ingestion. | ## Compatibility diff --git a/metadata-ingestion/archived/source_docs/ldap.md b/metadata-ingestion/archived/source_docs/ldap.md index b1644ec22319d6..4da535dc13808f 100644 --- a/metadata-ingestion/archived/source_docs/ldap.md +++ b/metadata-ingestion/archived/source_docs/ldap.md @@ -34,6 +34,25 @@ source: # Options base_dn: "dc=example,dc=org" + # Optional: Map LDAP User Attributes to DataHub User Attributes + user_attrs_map: + urn: sAMAccountName # A unique, stable ID for the User + fullName: cn + lastName: sn + firstName: givenName + displayName: displayName + manager: manager + mail: mail + departmentNumber: departmentNumber + title: title + + # Optional: Map LDAP Group Attributes to DataHub Group Attributes + group_attrs_map: + urn: cn # A unique, stable ID for the Group + admins: owner + members: uniqueMember + displayName: name + sink: # sink configs ``` @@ -42,20 +61,48 @@ sink: Note that a `.` is used to denote nested fields in the YAML recipe. -| Field | Required | Default | Description | -| ------------------------------ | -------- | ------------------- | ----------------------------------------------------------------------- | -| `ldap_server` | ✅ | | LDAP server URL. | -| `ldap_user` | ✅ | | LDAP user. | -| `ldap_password` | ✅ | | LDAP password. | -| `base_dn` | ✅ | | LDAP DN. | -| `filter` | | `"(objectClass=*)"` | LDAP extractor filter. | -| `drop_missing_first_last_name` | | `True` | If set to true, any users without first and last names will be dropped. | -| `page_size` | | `20` | Size of each page to fetch when extracting metadata. | +| Field | Required | Default | Description | +| ------------------------------- | -------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ldap_server` | ✅ | | LDAP server URL. | +| `ldap_user` | ✅ | | LDAP user. | +| `ldap_password` | ✅ | | LDAP password. | +| `base_dn` | ✅ | | LDAP DN. | +| `filter` | | `"(objectClass=*)"` | LDAP extractor filter. | +| `drop_missing_first_last_name` | | `True` | If set to true, any users without first and last names will be dropped. | +| `page_size` | | `20` | Size of each page to fetch when extracting metadata. | +| `user_attrs_map.urn` | | `sAMAccountName` | An attribute to use in constructing the DataHub User urn. This should be something that uniquely identifies the user and is stable over time. | +| `user_attrs_map.managerUrn` | | `manager` | Alternate attrs key representing same information as user's manager in the organization. | +| `user_attrs_map.firstName` | | `givenName` | Alternate attrs key representing same information as user's givenName in the organization. | +| `user_attrs_map.lastName` | | `sn` | Alternate attrs key representing same information as user's sn (surname) in the organization. | +| `user_attrs_map.fullName` | | `cn` | Alternate attrs key representing same information as user's cn (common name) in the organization. | +| `user_attrs_map.email` | | `mail` | Alternate attrs key representing same information as user's mail in the organization. | +| `user_attrs_map.displayName` | | `displayName` | Alternate attrs key representing same information as user's displayName in the organization. | +| `user_attrs_map.departmentId` | | `departmentNumber` | Alternate attrs key representing same information as user's departmentNumber in the organization. | +| `user_attrs_map.departmentName` | | `departmentNumber` | Alternate attrs key representing same information as user's departmentName in the organization. It is defaulted to `departmentNumber` to not impact existing users. New users are recommended to use descriptive attributes like `department` or `departmantName` that may exist. | +| `user_attrs_map.title` | | `title` | Alternate attrs key representing same information as user's title in the organization. | +| `user_attrs_map.countryCode` | | `countryCode` | Alternate attrs key representing same information as user's countryCode in the organization. | +| `group_attrs_map.urn` | | `cn` | Alternate attrs key representing same information as the group's cn (common name) for the LDAP group. | +| `group_attrs_map.email` | | `mail` | Alternate attrs key representing same information as group's mail in the organization. | +| `group_attrs_map.admins` | | `owner` | Alternate attrs key representing same information as group's owner in the organization. | +| `group_attrs_map.members` | | `uniqueMember` | Alternate attrs key representing same information as group's members in the organization. | +| `group_attrs_map.displayName` | | `name` | Alternate attrs key representing same information as group's display name in the organization. | +| `group_attrs_map.description` | | `info` | Alternate attrs key representing same information as group's description in the organization. | The `drop_missing_first_last_name` should be set to true if you've got many "headless" user LDAP accounts for devices or services should be excluded when they do not contain a first and last name. This will only impact the ingestion of LDAP users, while LDAP groups will be unaffected by this config option. +### Configurable LDAP + +Every organization may implement LDAP slightly differently based on their needs. The makes a standard LDAP recipe ineffective due to missing data during LDAP ingestion. For instance, LDAP recipe assumes department information for a CorpUser would be present in the `departmentNumber` attribute. If an organization chose not to implement that attribute or rather capture similar imformation in the `department` attribute, that information can be missed during LDAP ingestion (even though the information may be present in LDAP in a slightly different form). LDAP source provides flexibility to provide optional mapping for such variations to be reperesented under `user_attrs_map` and `group_attrs_map`. So if an organization represented `departmentNumber` as `department` and `mail` as `email`, the recipe can be adapted to customize that mapping based on need. An example is show below. If `user_attrs_map` section is not provided, the default mapping will apply. + +```yaml +# in config section +user_attrs_map: + departmentNumber: department + mail: email +``` + ## Compatibility Coming soon! diff --git a/metadata-ingestion/archived/source_docs/superset.md b/metadata-ingestion/archived/source_docs/superset.md index 58ea6f81ab44c2..abc76e9264c57d 100644 --- a/metadata-ingestion/archived/source_docs/superset.md +++ b/metadata-ingestion/archived/source_docs/superset.md @@ -64,12 +64,15 @@ Note that a `.` is used to denote nested fields in the YAML recipe. | Field | Required | Default | Description | | ------------- | -------- | ------------------ | ------------------------------------------------------- | | `connect_uri` | | `"localhost:8088"` | Superset host URL. | +| `display_uri` | | `(connect_uri)` | Publicly accessible Superset URL, see note below. | | `username` | | | Superset username. | | `password` | | | Superset password. | | `provider` | | `"db"` | Superset provider. | | `env` | | `"PROD"` | Environment to use in namespace when constructing URNs. | | `database_alias` | | | Can be used to change mapping for database names in superset to what you have in datahub | +NOTE: `display_uri` can be used when you need to ingest from a private, specially configured instance, but still want dashboard, graph, etc. links to point to the publicly accessible URL. So, for example, you could set `connect_uri: localhost:xxxx, display_uri: superset.mydomain.com`. You may need to do this if `superset.mydomain.com` has complex authentication that is not easy to pass through this source config. + ## Compatibility Coming soon! diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index 95a8b2a58e8df9..b26c468c33d867 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -65,13 +65,13 @@ task lint(type: Exec, dependsOn: installDev) { The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0: "venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax". */ - commandLine 'bash', '-x', '-c', + commandLine 'bash', '-c', "find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/ # type: List\\[str\\]//g' {} \\; && " + - "source ${venv_name}/bin/activate && black --check --diff src/ tests/ examples/ && isort --check --diff src/ tests/ examples/ && flake8 --count --statistics src/ tests/ examples/ && mypy src/ tests/ examples/" + "source ${venv_name}/bin/activate && set -x && black --check --diff src/ tests/ examples/ && isort --check --diff src/ tests/ examples/ && flake8 --count --statistics src/ tests/ examples/ && mypy src/ tests/ examples/" } task lintFix(type: Exec, dependsOn: installDev) { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && " + + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && set -x && " + "black src/ tests/ examples/ && " + "isort src/ tests/ examples/ && " + "flake8 src/ tests/ examples/ && " + @@ -83,15 +83,15 @@ task testQuick(type: Exec, dependsOn: installDev) { inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -m 'not integration and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && pytest --durations=20 -m 'not integration and not integration_batch_1 and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } -task installDevTest(type: Exec, dependsOn: [installDev]) { +task installDevTest(type: Exec, dependsOn: [install]) { inputs.file file('setup.py') outputs.dir("${venv_name}") outputs.file("${venv_name}/.build_install_dev_test_sentinel") - commandLine 'bash', '-x', '-c', + commandLine 'bash', '-c', "${venv_name}/bin/pip install -e .[dev,integration-tests] && touch ${venv_name}/.build_install_dev_test_sentinel" } @@ -99,9 +99,9 @@ def testFile = hasProperty('testFile') ? testFile : 'unknown' task testSingle(dependsOn: [installDevTest]) { doLast { if (testFile != 'unknown') { - exec { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest ${testFile}" + exec { + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && pytest ${testFile}" } } else { throw new GradleException("No file provided. Use -PtestFile=") @@ -109,14 +109,32 @@ task testSingle(dependsOn: [installDevTest]) { } } -task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { +task installAirflow1(type: Exec, dependsOn: [install]) { + inputs.file file('setup.py') + outputs.dir("${venv_name}") + outputs.file("${venv_name}/.build_install_airflow_sentinel") commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" + "${venv_name}/bin/pip install -e .[dev-airflow1] -c tests/airflow1-constraints.txt && touch ${venv_name}/.build_install_airflow_sentinel" } -task testSlowIntegration(type: Exec, dependsOn: [testQuick, installDevTest]) { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -m 'slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" +task testIntegration(type: Exec, dependsOn: [installDevTest]) { + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && pytest --durations=50 -m 'integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml" +} + +task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) { + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && pytest --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml" +} + +task testFull(type: Exec, dependsOn: [installDevTest]) { + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && pytest --durations=50 -vv --continue-on-collection-errors --junit-xml=junit.full.xml" +} + +task testSlowIntegration(type: Exec, dependsOn: [installDevTest]) { + commandLine 'bash', '-c', + "source ${venv_name}/bin/activate && pytest --durations=20 -m 'slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.slow.integration.xml" } task docGen(type: Exec, dependsOn: [codegen, installDevTest]) { @@ -126,8 +144,8 @@ task docGen(type: Exec, dependsOn: [codegen, installDevTest]) { task cleanPythonCache(type: Exec) { - commandLine 'bash', '-x', '-c', - "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete" + commandLine 'bash', '-c', + "find src tests -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete" } build.dependsOn install @@ -140,5 +158,8 @@ clean { delete 'dist' delete 'src/datahub/metadata' delete '../docs/generated' + delete 'generated' + delete '.mypy_cache' + delete '.pytest_cache' } clean.dependsOn cleanPythonCache diff --git a/metadata-ingestion/docs/sources/bigquery/bigquery.md b/metadata-ingestion/docs/sources/bigquery/bigquery.md index a140f8fc815854..5105586094d37e 100644 --- a/metadata-ingestion/docs/sources/bigquery/bigquery.md +++ b/metadata-ingestion/docs/sources/bigquery/bigquery.md @@ -3,6 +3,8 @@ 1. Create a custom role for datahub as per [BigQuery docs](https://cloud.google.com/iam/docs/creating-custom-roles#creating_a_custom_role) 2. Grant the following permissions to this role: ``` + # basic requirements + bigquery.datasets.get bigquery.datasets.getIamPolicy bigquery.jobs.create @@ -12,16 +14,21 @@ bigquery.models.list bigquery.routines.get bigquery.routines.list - bigquery.tables.create # Needs for profiling bigquery.tables.get - bigquery.tables.getData # Needs for profiling + resourcemanager.projects.get + bigquery.readsessions.create + bigquery.readsessions.getData + + # needed if profiling enabled + + bigquery.tables.create + bigquery.tables.getData bigquery.tables.list + # needed for lineage generation via GCP logging + logging.logEntries.list logging.privateLogEntries.list - resourcemanager.projects.get - bigquery.readsessions.create - bigquery.readsessions.getData ``` #### Create a service account @@ -89,5 +96,37 @@ Due to performance reasons, we only profile the latest partition for Partitioned You can set partition explicitly with `partition.partition_datetime` property if you want. (partition will be applied to all partitioned tables) ::: +### Working with multi-project GCP setups + +Sometimes you may have multiple GCP project with one only giving you view access rights and other project where you have view/modify rights. To deal with such setups you can use the `storage_project_id` setting. An example recipe looks like this + +```yaml +source: + type: "bigquery" + config: + project_id: compute-project-id # With view as well as modify rights + storage_project_id: acryl-staging # with view only rights + ...rest of fields +``` + +The GCP roles with which this setup has been tested are as follows +- Storage Project + - BigQuery Data Viewer + - BigQuery Metadata Viewer + - Logs Viewer + - Private Logs Viewer +- Compute Project + - BigQuery Admin + - BigQuery Data Editor + - BigQuery Job User + +If you are using `use_exported_bigquery_audit_metadata = True` and `use_v2_audit_metadata = False` then make sure you prefix the datasets in `bigquery_audit_metadata_datasets` with storage project id. + +:::note + +Bigquery usage has not been modified and tested with multi-project setting. Only `bigquery` plugin works with multi-project setup currently. + +:::note + ### Caveats -- For Materialized views lineage is dependent on logs being retained. If your GCP logging is retained for 30 days (default) and 30 days have passed since the creation of the materialized view we won't be able to get lineage for them. \ No newline at end of file +- For Materialized views lineage is dependent on logs being retained. If your GCP logging is retained for 30 days (default) and 30 days have passed since the creation of the materialized view we won't be able to get lineage for them. diff --git a/metadata-ingestion/docs/sources/dbt/dbt.md b/metadata-ingestion/docs/sources/dbt/dbt.md index 40a2513007213f..bd5ff9efeb3149 100644 --- a/metadata-ingestion/docs/sources/dbt/dbt.md +++ b/metadata-ingestion/docs/sources/dbt/dbt.md @@ -2,13 +2,49 @@ dbt allows authors to define meta properties for datasets. Checkout this link to know more - [dbt meta](https://docs.getdbt.com/reference/resource-configs/meta). Our dbt source allows users to define actions such as add a tag, term or owner. For example if a dbt model has a meta config ```"has_pii": True```, we can define an action that evaluates if the property is set to true and add, lets say, a ```pii``` tag. -To leverage this feature we require users to define mappings as part of the recipe. Following is how mappings can be defined - +To leverage this feature we require users to define mappings as part of the recipe. The following section describes how you can build these mappings. Listed below is a meta_mapping section that among other things, looks for keys like `business_owner` and adds owners that are listed there. + + + + +```yaml +meta_mapping: + business_owner: + match: ".*" + operation: "add_owner" + config: + owner_type: user + owner_category: BUSINESS_OWNER + has_pii: + match: True + operation: "add_tag" + config: + tag: "has_pii_test" + int_property: + match: 1 + operation: "add_tag" + config: + tag: "int_meta_property" + double_property: + match: 2.5 + operation: "add_term" + config: + term: "double_meta_property" + data_governance.team_owner: + match: "Finance" + operation: "add_term" + config: + term: "Finance_test" +``` + + + ```json "meta_mapping": { "business_owner": { "match": ".*", "operation": "add_owner", - "config": {"owner_type": "user"}, + "config": {"owner_type": "user", "owner_category": "BUSINESS_OWNER"}, }, "has_pii": { "match": True, @@ -32,11 +68,145 @@ To leverage this feature we require users to define mappings as part of the reci }, } ``` -We support the below actions - + + + +We support the following operations: 1. add_tag - Requires ```tag``` property in config. 2. add_term - Requires ```term``` property in config. -3. add_owner - Requires ```owner_type``` property in config which can be either user or group. +3. add_owner - Requires ```owner_type``` property in config which can be either user or group. Optionally accepts the ```owner_category``` config property which you can set to one of ```['TECHNICAL_OWNER', 'BUSINESS_OWNER', 'DATA_STEWARD', 'DATAOWNER'``` (defaults to `DATAOWNER`). Note: -1. Currently, dbt meta mapping is only supported for meta configs defined at the top most level or a node in manifest file. If that is not preset we will look for meta in the config section of the node. -2. For string based meta properties we support regex matching. \ No newline at end of file +1. Currently, dbt meta mapping is only supported for meta elements defined at the model level (not supported for columns). +2. For string meta properties we support regex matching. + +With regex matching, you can also use the matched value to customize how you populate the tag, term or owner fields. Here are a few advanced examples: + +#### Data Tier - Bronze, Silver, Gold + +If your meta section looks like this: +```yaml + meta: + data_tier: Bronze # chosen from [Bronze,Gold,Silver] +``` +and you wanted to attach a glossary term like `urn:li:glossaryTerm:Bronze` for all the models that have this value in the meta section attached to them, the following meta_mapping section would achieve that outcome: +```yaml +meta_mapping: + data_tier: + match: "Bronze|Silver|Gold" + operation: "add_term" + config: + term: "{{ $match }}" +``` +to match any data_tier of Bronze, Silver or Gold and maps it to a glossary term with the same name. + +#### Case Numbers - create tags +If your meta section looks like this: +```yaml + meta: + case: PLT-4678 # internal Case Number +``` +and you want to generate tags that look like `case_4678` from this, you can use the following meta_mapping section: +```yaml +meta_mapping: + case: + match: "PLT-(.*)" + operation: "add_tag" + config: + tag: "case_{{ $match }}" +``` + +#### Stripping out leading @ sign + +You can also match specific groups within the value to extract subsets of the matched value. e.g. if you have a meta section that looks like this: +```yaml +meta: + owner: "@finance-team" + business_owner: "@janet" +``` +and you want to mark the finance-team as a group that owns the dataset (skipping the leading @ sign), while marking janet as an individual user (again, skipping the leading @ sign) that owns the dataset, you can use the following meta-mapping section. +```yaml +meta_mapping: + owner: + match: "^@(.*)" + operation: "add_owner" + config: + owner_type: group + business_owner: + match: "^@(?P(.*))" + operation: "add_owner" + config: + owner_type: user + owner_category: BUSINESS_OWNER +``` +In the examples above, we show two ways of writing the matching regexes. In the first one, `^@(.*)` the first matching group (a.k.a. match.group(1)) is automatically inferred. In the second example, `^@(?P(.*))`, we use a named matching group (called owner, since we are matching an owner) to capture the string we want to provide to the ownership urn. + + +### dbt query_tag automated mappings +This works similarly as the dbt meta mapping but for the query tags + +We support the below actions - +1. add_tag - Requires ```tag``` property in config. + +The below example set as global tag the query tag `tag` key's value. +```json +"query_tag_mapping": +{ + "tag": + "match": ".*" + "operation": "add_tag" + "config": + "tag": "{{ $match }}" +} +``` + +### Integrating with dbt test + +To integrate with dbt tests, the `dbt` source needs access to the `run_results.json` file generated after a `dbt test` execution. Typically, this is written to the `target` directory. A common pattern you can follow is: +1. Run `dbt docs generate` and upload `manifest.json` and `catalog.json` to a location accessible to the `dbt` source (e.g. s3 or local file system) +2. Run `dbt test` and upload `run_results.json` to a location accessible to the `dbt` source (e.g. s3 or local file system) +3. Run `datahub ingest -c dbt_recipe.dhub.yaml` with the following config parameters specified + * test_results_path: pointing to the run_results.json file that you just created + +The connector will produce the following things: +- Assertion definitions that are attached to the dataset (or datasets) +- Results from running the tests attached to the timeline of the dataset + +#### View of dbt tests for a dataset +![test view](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/dbt-tests-view.png) +#### Viewing the SQL for a dbt test +![test logic view](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/dbt-test-logic-view.png) +#### Viewing timeline for a failed dbt test +![test view](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/dbt-tests-failure-view.png) + +#### Separating test result emission from other metadata emission + +You can segregate emission of test results from the emission of other dbt metadata using the `entities_enabled` config flag. +The following recipe shows you how to emit only test results. + +```yaml +source: + type: dbt + config: + manifest_path: _path_to_manifest_json + catalog_path: _path_to_catalog_json + test_results_path: _path_to_run_results_json + target_platform: postgres + entities_enabled: + test_results: Only +``` + +Similarly, the following recipe shows you how to emit everything (i.e. models, sources, seeds, test definitions) but not test results: +```yaml +source: + type: dbt + config: + manifest_path: _path_to_manifest_json + catalog_path: _path_to_catalog_json + run_results_path: _path_to_run_results_json + target_platform: postgres + entities_enabled: + test_results: No +``` + + diff --git a/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml b/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml index 6f660a8d685db0..4ee236b74b934b 100644 --- a/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml +++ b/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml @@ -2,13 +2,14 @@ source: type: "dbt" config: # Coordinates - manifest_path: "./path/dbt/manifest_file.json" - catalog_path: "./path/dbt/catalog_file.json" - sources_path: "./path/dbt/sources_file.json" + # To use this as-is, set the environment variable DBT_PROJECT_ROOT to the root folder of your dbt project + manifest_path: "${DBT_PROJECT_ROOT}/target/manifest_file.json" + catalog_path: "${DBT_PROJECT_ROOT}/target/catalog_file.json" + sources_path: "${DBT_PROJECT_ROOT}/target/sources_file.json" # optional for freshness + test_results_path: "${DBT_PROJECT_ROOT}/target/run_results.json" # optional for recording dbt test results after running dbt test # Options target_platform: "my_target_platform_id" # e.g. bigquery/postgres/etc. - load_schemas: True # note: if this is disabled + load_schemas: False # note: enable this only if you are not ingesting metadata from your warehouse -sink: - # sink configs \ No newline at end of file +# sink configs diff --git a/metadata-ingestion/docs/sources/delta-lake/delta-lake.md b/metadata-ingestion/docs/sources/delta-lake/delta-lake.md new file mode 100644 index 00000000000000..9c620d9840e340 --- /dev/null +++ b/metadata-ingestion/docs/sources/delta-lake/delta-lake.md @@ -0,0 +1,142 @@ +## Usage Guide + +If you are new to [Delta Lake](https://delta.io/) and want to test out a simple integration with Delta Lake and DataHub, you can follow this guide. + +### Delta Table on Local File System + +#### Step 1 +Create a delta table using the sample PySpark code below if you don't have a delta table you can point to. + +```python +import uuid +import random +from pyspark.sql import SparkSession +from delta.tables import DeltaTable + +def generate_data(): + return [(y, m, d, str(uuid.uuid4()), str(random.randrange(10000) % 26 + 65) * 3, random.random()*10000) + for d in range(1, 29) + for m in range(1, 13) + for y in range(2000, 2021)] + +jar_packages = ["org.apache.hadoop:hadoop-aws:3.2.3", "io.delta:delta-core_2.12:1.2.1"] +spark = SparkSession.builder \ + .appName("quickstart") \ + .master("local[*]") \ + .config("spark.jars.packages", ",".join(jar_packages)) \ + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ + .getOrCreate() + +table_path = "quickstart/my-table" +columns = ["year", "month", "day", "sale_id", "customer", "total_cost"] +spark.sparkContext.parallelize(generate_data()).toDF(columns).repartition(1).write.format("delta").save(table_path) + +df = spark.read.format("delta").load(table_path) +df.show() + +``` + +#### Step 2 +Create a datahub ingestion yaml file (delta.dhub.yaml) to ingest metadata from the delta table you just created. + +```yaml +source: + type: "delta-lake" + config: + base_path: "quickstart/my-table" + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +Note: Make sure you run the Spark code as well as recipe from same folder otherwise use absolute paths. + +#### Step 3 +Execute the ingestion recipe: +```shell +datahub ingest -c delta.dhub.yaml +``` + +### Delta Table on S3 + +#### Step 1 +Set up your AWS credentials by creating an AWS credentials config file; typically in '$HOME/.aws/credentials'. +``` +[my-creds] +aws_access_key_id: ###### +aws_secret_access_key: ###### +``` +Step 2: Create a Delta Table using the PySpark sample code below unless you already have Delta Tables on your S3. +```python +from pyspark.sql import SparkSession +from delta.tables import DeltaTable +from configparser import ConfigParser +import uuid +import random +def generate_data(): + return [(y, m, d, str(uuid.uuid4()), str(random.randrange(10000) % 26 + 65) * 3, random.random()*10000) + for d in range(1, 29) + for m in range(1, 13) + for y in range(2000, 2021)] + +jar_packages = ["org.apache.hadoop:hadoop-aws:3.2.3", "io.delta:delta-core_2.12:1.2.1"] +spark = SparkSession.builder \ + .appName("quickstart") \ + .master("local[*]") \ + .config("spark.jars.packages", ",".join(jar_packages)) \ + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ + .getOrCreate() + + +config_object = ConfigParser() +config_object.read("$HOME/.aws/credentials") +profile_info = config_object["my-creds"] +access_id = profile_info["aws_access_key_id"] +access_key = profile_info["aws_secret_access_key"] + +hadoop_conf = spark._jsc.hadoopConfiguration() +hadoop_conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") +hadoop_conf.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") +hadoop_conf.set("fs.s3a.access.key", access_id) +hadoop_conf.set("fs.s3a.secret.key", access_key) + +table_path = "s3a://my-bucket/my-folder/sales-table" +columns = ["year", "month", "day", "sale_id", "customer", "total_cost"] +spark.sparkContext.parallelize(generate_data()).toDF(columns).repartition(1).write.format("delta").save(table_path) +df = spark.read.format("delta").load(table_path) +df.show() + +``` + +#### Step 3 +Create a datahub ingestion yaml file (delta.s3.dhub.yaml) to ingest metadata from the delta table you just created. + +```yml +source: + type: "delta-lake" + config: + base_path: "s3://my-bucket/my-folder/sales-table" + s3: + aws_config: + aws_access_key_id: <> + aws_secret_access_key: <> + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +#### Step 4 +Execute the ingestion recipe: +```shell +datahub ingest -c delta.s3.dhub.yaml +``` + +### Note + +The above recipes are minimal recipes. Please refer to [Config Details](#config-details) section for the full configuration. diff --git a/metadata-ingestion/docs/sources/delta-lake/delta-lake_recipe.yml b/metadata-ingestion/docs/sources/delta-lake/delta-lake_recipe.yml new file mode 100644 index 00000000000000..fac8dc4f637415 --- /dev/null +++ b/metadata-ingestion/docs/sources/delta-lake/delta-lake_recipe.yml @@ -0,0 +1,9 @@ +source: + type: delta-lake + config: + env: "PROD" + platform_instance: "my-delta-lake" + base_path: "/path/to/data/folder" + +sink: + # sink configs \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/elastic-search/elasticsearch_recipe.yml b/metadata-ingestion/docs/sources/elastic-search/elasticsearch_recipe.yml index 689f1fb1962e60..d93588153fd2ed 100644 --- a/metadata-ingestion/docs/sources/elastic-search/elasticsearch_recipe.yml +++ b/metadata-ingestion/docs/sources/elastic-search/elasticsearch_recipe.yml @@ -23,6 +23,9 @@ source: index_pattern: allow: [".*some_index_name_pattern*"] deny: [".*skip_index_name_pattern*"] + ingest_index_templates: False + index_template_pattern: + allow: [".*some_index_template_name_pattern*"] sink: # sink configs diff --git a/metadata-ingestion/docs/sources/hive/hive_recipe.yml b/metadata-ingestion/docs/sources/hive/hive_recipe.yml index 091f161936966f..f84659016dc451 100644 --- a/metadata-ingestion/docs/sources/hive/hive_recipe.yml +++ b/metadata-ingestion/docs/sources/hive/hive_recipe.yml @@ -59,8 +59,8 @@ source: type: hive config: host_port: :443 - username: token - password: + username: token / username + password: / password scheme: 'databricks+pyhive' options: diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi.md b/metadata-ingestion/docs/sources/powerbi/powerbi.md index 5e949e614bd5c7..2ef51dbd047cef 100644 --- a/metadata-ingestion/docs/sources/powerbi/powerbi.md +++ b/metadata-ingestion/docs/sources/powerbi/powerbi.md @@ -1,3 +1,10 @@ +## Configuration Notes +See the +1. [Microsoft AD App Creation doc](https://docs.microsoft.com/en-us/power-bi/developer/embedded/embed-service-principal) for the steps to create an app client ID and secret +2. Login to Power BI as Admin and from Tenant settings allow below permissions +- Allow service principles to use Power BI APIs +- Allow service principals to use read-only Power BI admin APIs +- Enhance admin APIs responses with detailed metadata ## Concept mapping | Power BI | Datahub | diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml b/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml index 302fabc51a2fdf..935a8ce6e4b9be 100644 --- a/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml +++ b/metadata-ingestion/docs/sources/powerbi/powerbi_recipe.yml @@ -5,17 +5,18 @@ source: tenant_id: a949d688-67c0-4bf1-a344-e939411c6c0a # Ingest elements of below PowerBi Workspace into Datahub workspace_id: 4bd10256-e999-45dd-8e56-571c77153a5f - # Workspace's dataset environments (PROD, DEV, QA, STAGE) + # Workspaces dataset environments (PROD, DEV, QA, STAGE) env: DEV - # Azure AD App client identifier + # Azure AD Application identifier client_id: foo # Azure AD App client secret client_secret: bar + # Enable / Disable ingestion of user information for dashboards + extract_ownership: true # dataset_type_mapping is fixed mapping of Power BI datasources type to equivalent Datahub "data platform" dataset dataset_type_mapping: PostgreSql: postgres Oracle: oracle - sink: - # sink configs \ No newline at end of file + # sink configs diff --git a/metadata-ingestion/docs/sources/salesforce/salesforce.md b/metadata-ingestion/docs/sources/salesforce/salesforce.md new file mode 100644 index 00000000000000..61963c8da8b99a --- /dev/null +++ b/metadata-ingestion/docs/sources/salesforce/salesforce.md @@ -0,0 +1,30 @@ +### Prerequisites + +In order to ingest metadata from Salesforce, you will need: + +- Salesforce username, password, [security token](https://developer.Salesforce.com/docs/atlas.en-us.api.meta/api/sforce_api_concepts_security.htm) OR +- Salesforce instance url and access token/session id (suitable for one-shot ingestion only, as access token typically expires after 2 hours of inactivity) + +## Integration Details +This plugin extracts Salesforce Standard and Custom Objects and their details (fields, record count, etc) from a Salesforce instance. +Python library [simple-salesforce](https://pypi.org/project/simple-salesforce/) is used for authenticating and calling [Salesforce REST API](https://developer.Salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_what_is_rest_api.htm) to retrive details from Salesforce instance. + +### REST API Resources used in this integration +- [Versions](https://developer.Salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_versions.htm) +- [Tooling API Query](https://developer.salesforce.com/docs/atlas.en-us.api_tooling.meta/api_tooling/intro_rest_resources.htm) on objects EntityDefinition, EntityParticle, CustomObject, CustomField +- [Record Count](https://developer.Salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_record_count.htm) + +### Concept Mapping + +This ingestion source maps the following Source System Concepts to DataHub Concepts: + +| Source Concept | DataHub Concept | Notes | +| -- | -- | -- | +| `Salesforce` | [Data Platform](../../metamodel/entities/dataPlatform.md) | | +|Standard Object | [Dataset](../../metamodel/entities/dataset.md) | subtype "Standard Object" | +|Custom Object | [Dataset](../../metamodel/entities/dataset.md) | subtype "Custom Object" | + +### Caveats +- This connector has only been tested with Salesforce Developer Edition. +- This connector only supports table level profiling (Row and Column counts) as of now. Row counts are approximate as returned by [Salesforce RecordCount REST API](https://developer.Salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_record_count.htm). +- This integration does not support ingesting Salesforce [External Objects](https://developer.Salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_external_objects.htm) \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/salesforce/salesforce_recipe.yml b/metadata-ingestion/docs/sources/salesforce/salesforce_recipe.yml new file mode 100644 index 00000000000000..4fa50fd2606447 --- /dev/null +++ b/metadata-ingestion/docs/sources/salesforce/salesforce_recipe.yml @@ -0,0 +1,25 @@ +pipeline_name: my_salesforce_pipeline +source: + type: "salesforce" + config: + instance_url: "https://mydomain.my.salesforce.com/" + username: user@company + password: password_for_user + security_token: security_token_for_user + platform_instance: mydomain-dev-ed + domain: + sales: + allow: + - "Opportunity$" + - "Lead$" + + object_pattern: + allow: + - "Account$" + - "Opportunity$" + - "Lead$" + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/tableau/tableau.md b/metadata-ingestion/docs/sources/tableau/tableau.md index c58f160512ca33..a9200e305e0877 100644 --- a/metadata-ingestion/docs/sources/tableau/tableau.md +++ b/metadata-ingestion/docs/sources/tableau/tableau.md @@ -44,7 +44,7 @@ Workbooks from Tableau are ingested as Container in datahub.
- GraphQL query
```graphql { - workbooksConnection(first: 15, offset: 0, filter: {projectNameWithin: ["default", "Project 2"]}) { + workbooksConnection(first: 10, offset: 0, filter: {projectNameWithin: ["default", "Project 2"]}) { nodes { id name @@ -73,7 +73,7 @@ Dashboards from Tableau are ingested as Dashboard in datahub.
- GraphQL query
```graphql { - workbooksConnection(first: 15, offset: 0, filter: {projectNameWithin: ["default", "Project 2"]}) { + workbooksConnection(first: 10, offset: 0, filter: {projectNameWithin: ["default", "Project 2"]}) { nodes { ..... dashboards { @@ -185,7 +185,7 @@ Embedded Data source from Tableau is ingested as a Dataset in datahub. - GraphQL query
```graphql { - workbooksConnection(first: 15, offset: 0, filter: {projectNameWithin: ["default"]}) { + workbooksConnection(first: 10, offset: 0, filter: {projectNameWithin: ["default"]}) { nodes { .... embeddedDatasources { @@ -265,7 +265,7 @@ Published Data source from Tableau is ingested as a Dataset in datahub. - GraphQL query
```graphql { - publishedDatasourcesConnection(filter: {idWithin: ["00cce29f-b561-bb41-3557-8e19660bb5dd", "618c87db-5959-338b-bcc7-6f5f4cc0b6c6"]}) { + publishedDatasourcesConnection(first: 10, offset: 0, filter: {idWithin: ["00cce29f-b561-bb41-3557-8e19660bb5dd", "618c87db-5959-338b-bcc7-6f5f4cc0b6c6"]}) { nodes { __typename id @@ -343,7 +343,7 @@ For custom sql data sources, the query is viewable in UI under View Definition t - GraphQL query
```graphql { - customSQLTablesConnection(filter: {idWithin: ["22b0b4c3-6b85-713d-a161-5a87fdd78f40"]}) { + customSQLTablesConnection(first: 10, offset: 0, filter: {idWithin: ["22b0b4c3-6b85-713d-a161-5a87fdd78f40"]}) { nodes { id name @@ -408,8 +408,8 @@ Lineage is emitted as received from Tableau's metadata API for ## Troubleshooting -### Why are only some workbooks ingested from the specified project? +### Why are only some workbooks/custom SQLs/published datasources ingested from the specified project? This may happen when the Tableau API returns NODE_LIMIT_EXCEEDED error in response to metadata query and returns partial results with message "Showing partial results. , The request exceeded the ‘n’ node limit. Use pagination, additional filtering, or both in the query to adjust results." To resolve this, consider -- reducing the page size using the `workbooks_page_size` config param in datahub recipe (Defaults to 10). +- reducing the page size using the `page_size` config param in datahub recipe (Defaults to 10). - increasing tableau configuration [metadata query node limit](https://help.tableau.com/current/server/en-us/cli_configuration-set_tsm.htm#metadata_nodelimit) to higher value. \ No newline at end of file diff --git a/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/01-operation/ecommerce/01_snowflake_load.py b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/01-operation/ecommerce/01_snowflake_load.py new file mode 100644 index 00000000000000..582002f8d80f16 --- /dev/null +++ b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/01-operation/ecommerce/01_snowflake_load.py @@ -0,0 +1,38 @@ +import pendulum +from airflow.models import DAG +from airflow.operators.bash import BashOperator + +from datahub.api.graphql.operation import Operation +from datahub_provider.entities import Dataset +from datahub_provider.hooks.datahub import DatahubRestHook + +dag = DAG( + dag_id="snowflake_load", + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + schedule_interval="0 0 * * *", + catchup=False, +) + + +# Operation push +# The number of rows is hardcoded in this example but this shouldn't in normal operation +def report_operation(context): + hook: DatahubRestHook = DatahubRestHook("datahub_longtail") + host, password, timeout_sec = hook._get_config() + reporter = Operation(datahub_host=host, datahub_token=password, timeout=timeout_sec) + task = context["ti"].task + for outlet in task._outlets: + print(f"Reporting insert operation for {outlet.urn}") + reporter.report_operation( + urn=outlet.urn, operation_type="INSERT", num_affected_rows=123 + ) + + +pet_profiles_load = BashOperator( + task_id="load_s3_adoption_pet_profiles", + dag=dag, + inlets=[Dataset("s3", "longtail-core-data/mongo/adoption/pet_profiles")], + outlets=[Dataset("snowflake", "long_tail_companions.adoption.pet_profiles")], + bash_command="echo Dummy Task", + on_success_callback=report_operation, +) diff --git a/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/01-operation/marketing/01_send_emails.py b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/01-operation/marketing/01_send_emails.py new file mode 100644 index 00000000000000..62bf8962bfa721 --- /dev/null +++ b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/01-operation/marketing/01_send_emails.py @@ -0,0 +1,36 @@ +import datetime + +import pendulum +from airflow.models import DAG +from airflow.operators.bash import BashOperator + +from datahub_provider.entities import Dataset +from datahub_provider.operators.datahub_operation_sensor import ( + DataHubOperationCircuitBreakerSensor, +) + +dag = DAG( + dag_id="marketing-send_emails", + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + schedule_interval="0 0 * * *", + catchup=False, +) + +# New DataHub Operation Circuit Breaker Sensor +pet_profiles_operation_sensor = DataHubOperationCircuitBreakerSensor( + task_id="pet_profiles_operation_sensor", + datahub_rest_conn_id="datahub_longtail", + urn=[ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pet_profiles,PROD)" + ], + time_delta=datetime.timedelta(minutes=10), +) + +send_email = BashOperator( + task_id="send_emails", + dag=dag, + inlets=[Dataset("snowflake", "long_tail_companions.adoption.pet_profiles")], + bash_command="echo Dummy Task", +) + +pet_profiles_operation_sensor.set_downstream(send_email) diff --git a/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/02-assertion/ecommerce/02_snowflake_load.py b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/02-assertion/ecommerce/02_snowflake_load.py new file mode 100644 index 00000000000000..1adcee76178a45 --- /dev/null +++ b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/02-assertion/ecommerce/02_snowflake_load.py @@ -0,0 +1,42 @@ +import pendulum +from airflow.models import DAG +from airflow.operators.bash import BashOperator + +from datahub.api.graphql.operation import Operation +from datahub_provider.entities import Dataset +from datahub_provider.hooks.datahub import DatahubRestHook + +dag = DAG( + dag_id="snowflake_load", + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + schedule_interval="0 0 * * *", + catchup=False, +) + + +def report_operation(context): + hook: DatahubRestHook = DatahubRestHook("datahub_longtail") + host, password, timeout_sec = hook._get_config() + reporter = Operation(datahub_host=host, datahub_token=password, timeout=timeout_sec) + task = context["ti"].task + for inlet in task._outlets: + reporter.report_operation(urn=inlet.urn, operation_type="INSERT") + + +pet_profiles_load = BashOperator( + task_id="load_s3_adoption_pet_profiles", + dag=dag, + inlets=[Dataset("s3", "longtail-core-data/mongo/adoption/pet_profiles")], + outlets=[Dataset("snowflake", "long_tail_companions.adoption.pet_profiles")], + bash_command="echo Dummy Task", + on_success_callback=report_operation, +) + +# Simple bash command as example to load great expectation tests +run_ge_tests = BashOperator( + task_id="pet_profiles_ge_tests_run", + inlets=[Dataset("snowflake", "long_tail_companions.adoption.pet_profiles")], + bash_command="echo /usr/local/airflow/.local/bin/great_expectations checkpoint run pet_profiles", +) + +pet_profiles_load.set_downstream(run_ge_tests) diff --git a/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/02-assertion/marketing/02_send_emails.py b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/02-assertion/marketing/02_send_emails.py new file mode 100644 index 00000000000000..8f6464206e62a9 --- /dev/null +++ b/metadata-ingestion/examples/airflow/circuit_breaker/long_tail_companion/02-assertion/marketing/02_send_emails.py @@ -0,0 +1,52 @@ +import datetime + +import pendulum +from airflow.models import DAG +from airflow.operators.bash import BashOperator + +from datahub_provider.entities import Dataset +from datahub_provider.operators.datahub_assertion_operator import ( + DataHubAssertionOperator, +) +from datahub_provider.operators.datahub_operation_sensor import ( + DataHubOperationCircuitBreakerSensor, +) + +dag = DAG( + dag_id="marketing-send_emails", + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + schedule_interval="0 0 * * *", + catchup=False, +) + +items_operation_sensor = DataHubOperationCircuitBreakerSensor( + dag=dag, + task_id="pet_profiles_operation_sensor", + datahub_rest_conn_id="datahub_longtail", + urn=[ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pet_profiles,PROD)" + ], + time_delta=datetime.timedelta(days=1), +) + +# Assertion circuit breaker to check if there are assertions for the urns specified. +# verify_after_last_update is enabled which means it will get from the latest operation the timeframe +# it accepts assertions. +assertion_circuit_breaker = DataHubAssertionOperator( + task_id="pet_profiles_assertion_circuit_breaker", + datahub_rest_conn_id="datahub_longtail", + urn=[ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pet_profiles,PROD)" + ], + check_last_assertion_time=True, +) + +send_email = BashOperator( + task_id="send_emails", + dag=dag, + inlets=[Dataset("snowflake", "long_tail_companions.adoption.pet_profiles")], + bash_command="echo Dummy Task", +) + +items_operation_sensor.set_downstream(assertion_circuit_breaker) +assertion_circuit_breaker.set_downstream(send_email) diff --git a/metadata-ingestion/examples/demo_data/csv_enricher_demo_data.csv b/metadata-ingestion/examples/demo_data/csv_enricher_demo_data.csv new file mode 100644 index 00000000000000..d53f68f21858d8 --- /dev/null +++ b/metadata-ingestion/examples/demo_data/csv_enricher_demo_data.csv @@ -0,0 +1,4 @@ +resource,subresource,glossary_terms,tags,owners,ownership_type,description +"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",,[urn:li:glossaryTerm:CustomerAccount],[urn:li:tag:Legacy],[urn:li:corpuser:datahub|urn:li:corpuser:jdoe],TECHNICAL_OWNER,new description,Engineering +"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",field_foo,[urn:li:glossaryTerm:AccountBalance],,,,field_foo! +"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",field_bar,,[urn:li:tag:Legacy],,,field_bar? \ No newline at end of file diff --git a/metadata-ingestion/examples/library/dashboard_usage.py b/metadata-ingestion/examples/library/dashboard_usage.py new file mode 100644 index 00000000000000..ad3bd7c687b16c --- /dev/null +++ b/metadata-ingestion/examples/library/dashboard_usage.py @@ -0,0 +1,164 @@ +# Imports for urn construction utility methods +from datetime import datetime +from typing import List + +from datahub.emitter.mce_builder import make_dashboard_urn, make_user_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Imports for metadata model classes +from datahub.metadata.schema_classes import ( + CalendarIntervalClass, + ChangeTypeClass, + DashboardUsageStatisticsClass, + DashboardUserUsageCountsClass, + TimeWindowSizeClass, +) + +# Create rest emitter +rest_emitter = DatahubRestEmitter(gms_server="http://localhost:8080") + +usage_day_1_user_counts: List[DashboardUserUsageCountsClass] = [ + DashboardUserUsageCountsClass( + user=make_user_urn("user1"), executionsCount=3, usageCount=3 + ), + DashboardUserUsageCountsClass( + user=make_user_urn("user2"), executionsCount=2, usageCount=2 + ), +] + +usage_day_1: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType="dashboard", + changeType=ChangeTypeClass.UPSERT, + entityUrn=make_dashboard_urn("looker", "dashboards.999999"), + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.strptime("2022-02-09", "%Y-%m-%d").timestamp() * 1000 + ), + eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY), + uniqueUserCount=2, + executionsCount=5, + userCounts=usage_day_1_user_counts, + ), +) + +absolute_usage_as_of_day_1: MetadataChangeProposalWrapper = ( + MetadataChangeProposalWrapper( + entityType="dashboard", + changeType=ChangeTypeClass.UPSERT, + entityUrn=make_dashboard_urn("looker", "dashboards.999999"), + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.strptime("2022-02-09", "%Y-%m-%d").timestamp() * 1000 + ), + favoritesCount=100, + viewsCount=25, + lastViewedAt=round( + datetime.strptime( + "2022-02-09 04:45:30", "%Y-%m-%d %H:%M:%S" + ).timestamp() + * 1000 + ), + ), + ) +) + +rest_emitter.emit(usage_day_1) +rest_emitter.emit(absolute_usage_as_of_day_1) + +usage_day_2_user_counts: List[DashboardUserUsageCountsClass] = [ + DashboardUserUsageCountsClass( + user=make_user_urn("user1"), executionsCount=4, usageCount=4 + ), + DashboardUserUsageCountsClass( + user=make_user_urn("user2"), executionsCount=6, usageCount=6 + ), +] +usage_day_2: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType="dashboard", + changeType=ChangeTypeClass.UPSERT, + entityUrn=make_dashboard_urn("looker", "dashboards.999999"), + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.strptime("2022-02-10", "%Y-%m-%d").timestamp() * 1000 + ), + eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY), + uniqueUserCount=2, + executionsCount=10, + userCounts=usage_day_2_user_counts, + ), +) + +absolute_usage_as_of_day_2: MetadataChangeProposalWrapper = ( + MetadataChangeProposalWrapper( + entityType="dashboard", + changeType=ChangeTypeClass.UPSERT, + entityUrn=make_dashboard_urn("looker", "dashboards.999999"), + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.strptime("2022-02-10", "%Y-%m-%d").timestamp() * 1000 + ), + favoritesCount=100, + viewsCount=27, + lastViewedAt=round( + datetime.strptime( + "2022-02-10 10:45:30", "%Y-%m-%d %H:%M:%S" + ).timestamp() + * 1000 + ), + ), + ) +) + +rest_emitter.emit(usage_day_2) +rest_emitter.emit(absolute_usage_as_of_day_2) + +usage_day_3_user_counts: List[DashboardUserUsageCountsClass] = [ + DashboardUserUsageCountsClass( + user=make_user_urn("user1"), executionsCount=2, usageCount=2 + ), +] +usage_day_3: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType="dashboard", + changeType=ChangeTypeClass.UPSERT, + entityUrn=make_dashboard_urn("looker", "dashboards.999999"), + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.strptime("2022-02-11", "%Y-%m-%d").timestamp() * 1000 + ), + eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY), + uniqueUserCount=1, + executionsCount=2, + userCounts=usage_day_3_user_counts, + ), +) + +absolute_usage_as_of_day_3: MetadataChangeProposalWrapper = ( + MetadataChangeProposalWrapper( + entityType="dashboard", + changeType=ChangeTypeClass.UPSERT, + entityUrn=make_dashboard_urn("looker", "dashboards.999999"), + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.strptime("2022-02-11", "%Y-%m-%d").timestamp() * 1000 + ), + favoritesCount=102, + viewsCount=30, + lastViewedAt=round( + datetime.strptime( + "2022-02-11 02:45:30", "%Y-%m-%d %H:%M:%S" + ).timestamp() + * 1000 + ), + ), + ) +) + +rest_emitter.emit(usage_day_3) +rest_emitter.emit(absolute_usage_as_of_day_3) diff --git a/metadata-ingestion/examples/library/data_quality_mcpw_rest.py b/metadata-ingestion/examples/library/data_quality_mcpw_rest.py index 7672d634f58468..077ca550e880eb 100644 --- a/metadata-ingestion/examples/library/data_quality_mcpw_rest.py +++ b/metadata-ingestion/examples/library/data_quality_mcpw_rest.py @@ -47,7 +47,7 @@ def emitAssertionResult(assertionResult: AssertionRunEvent) -> None: aspect=assertionResult, ) - # Emit BatchAssertion Result! (timseries aspect) + # Emit BatchAssertion Result! (timeseries aspect) emitter.emit_mcp(dataset_assertionRunEvent_mcp) diff --git a/metadata-ingestion/examples/library/dataset_add_column_tag.py b/metadata-ingestion/examples/library/dataset_add_column_tag.py index f5243ce28a5f01..8a15d33ff78779 100644 --- a/metadata-ingestion/examples/library/dataset_add_column_tag.py +++ b/metadata-ingestion/examples/library/dataset_add_column_tag.py @@ -23,18 +23,15 @@ def get_simple_field_path_from_v2_field_path(field_path: str) -> str: """A helper function to extract simple . path notation from the v2 field path""" - if field_path.startswith("[version=2.0]"): - # this is a v2 field path - tokens = [ - t - for t in field_path.split(".") - if not (t.startswith("[") or t.endswith("]")) - ] - path = ".".join(tokens) - return path - else: + if not field_path.startswith("[version=2.0]"): # not a v2, we assume this is a simple path return field_path + # this is a v2 field path + tokens = [ + t for t in field_path.split(".") if not (t.startswith("[") or t.endswith("]")) + ] + + return ".".join(tokens) # Inputs -> the column, dataset and the tag to set diff --git a/metadata-ingestion/examples/library/dataset_add_column_term.py b/metadata-ingestion/examples/library/dataset_add_column_term.py index ff1cad48a9f0c0..d656b5bd4502e7 100644 --- a/metadata-ingestion/examples/library/dataset_add_column_term.py +++ b/metadata-ingestion/examples/library/dataset_add_column_term.py @@ -23,18 +23,15 @@ def get_simple_field_path_from_v2_field_path(field_path: str) -> str: """A helper function to extract simple . path notation from the v2 field path""" - if field_path.startswith("[version=2.0]"): - # this is a v2 field path - tokens = [ - t - for t in field_path.split(".") - if not (t.startswith("[") or t.endswith("]")) - ] - path = ".".join(tokens) - return path - else: + if not field_path.startswith("[version=2.0]"): # not a v2, we assume this is a simple path return field_path + # this is a v2 field path + tokens = [ + t for t in field_path.split(".") if not (t.startswith("[") or t.endswith("]")) + ] + + return ".".join(tokens) # Inputs -> the column, dataset and the term to set diff --git a/metadata-ingestion/examples/library/dataset_query_entity_v2.py b/metadata-ingestion/examples/library/dataset_query_entity_v2.py new file mode 100644 index 00000000000000..a8b8439079f22c --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_query_entity_v2.py @@ -0,0 +1,33 @@ +import logging + +from datahub.emitter.mce_builder import make_dataset_urn + +# read-modify-write requires access to the DataHubGraph (RestEmitter is not enough) +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +# Imports for metadata model classes +from datahub.metadata.schema_classes import ( + DataPlatformInstanceClass, + DatasetKeyClass, + StatusClass, +) + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +dataset_urn = make_dataset_urn(platform="hive", name="realestate_db.sales", env="PROD") + +gms_endpoint = "http://localhost:8080" +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) + +# Query multiple aspects from entity +result = graph.get_aspects_for_entity( + entity_urn=dataset_urn, + aspects=["status", "dataPlatformInstance", "datasetKey"], + aspect_types=[StatusClass, DataPlatformInstanceClass, DatasetKeyClass], +) + +# result are typed according to their class if exist +if result is not None: + if result["datasetKey"]: + log.info(result["datasetKey"].name) diff --git a/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py b/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py index 11f73d36cb29d9..d1c934cba40409 100644 --- a/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py +++ b/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py @@ -10,13 +10,11 @@ ) from datahub.metadata.schema_classes import ChangeTypeClass -# Construct upstream tables. -upstream_tables: List[UpstreamClass] = [] upstream_table_1 = UpstreamClass( dataset=builder.make_dataset_urn("bigquery", "upstream_table_1", "PROD"), type=DatasetLineageTypeClass.TRANSFORMED, ) -upstream_tables.append(upstream_table_1) +upstream_tables: List[UpstreamClass] = [upstream_table_1] upstream_table_2 = UpstreamClass( dataset=builder.make_dataset_urn("bigquery", "upstream_table_2", "PROD"), type=DatasetLineageTypeClass.TRANSFORMED, diff --git a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py index 7212282156d8b9..1871a8af09e50c 100644 --- a/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py +++ b/metadata-ingestion/examples/library/lineage_job_dataflow_new_api_simple.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime +from datetime import datetime, timezone from datahub.api.entities.datajob import DataFlow, DataJob from datahub.api.entities.dataprocess.dataprocess_instance import ( @@ -36,40 +36,61 @@ jobFlowRun = DataProcessInstance.from_dataflow( dataflow=jobFlow, id=f"{jobFlow.id}-{uuid.uuid4()}" ) -jobFlowRun.emit_process_start(emitter, int(datetime.utcnow().timestamp() * 1000)) +jobFlowRun.emit_process_start( + emitter, int(datetime.now(timezone.utc).timestamp() * 1000) +) + jobRun = DataProcessInstance.from_datajob( datajob=dataJob, id=f"{jobFlow.id}-{uuid.uuid4()}" ) -jobRun.emit_process_start(emitter, int(datetime.utcnow().timestamp() * 1000)) +jobRun.emit_process_start(emitter, int(datetime.now(timezone.utc).timestamp() * 1000)) + jobRun.emit_process_end( - emitter, int(datetime.utcnow().timestamp() * 1000), result=InstanceRunResult.SUCCESS + emitter, + int(datetime.now(timezone.utc).timestamp() * 1000), + result=InstanceRunResult.SUCCESS, ) + job2Run = DataProcessInstance.from_datajob( datajob=dataJob2, id=f"{jobFlow.id}-{uuid.uuid4()}" ) -job2Run.emit_process_start(emitter, int(datetime.utcnow().timestamp() * 1000)) +job2Run.emit_process_start(emitter, int(datetime.now(timezone.utc).timestamp() * 1000)) + job2Run.emit_process_end( - emitter, int(datetime.utcnow().timestamp() * 1000), result=InstanceRunResult.SUCCESS + emitter, + int(datetime.now(timezone.utc).timestamp() * 1000), + result=InstanceRunResult.SUCCESS, ) + job3Run = DataProcessInstance.from_datajob( datajob=dataJob3, id=f"{jobFlow.id}-{uuid.uuid4()}" ) -job3Run.emit_process_start(emitter, int(datetime.utcnow().timestamp() * 1000)) +job3Run.emit_process_start(emitter, int(datetime.now(timezone.utc).timestamp() * 1000)) + job3Run.emit_process_end( - emitter, int(datetime.utcnow().timestamp() * 1000), result=InstanceRunResult.SUCCESS + emitter, + int(datetime.now(timezone.utc).timestamp() * 1000), + result=InstanceRunResult.SUCCESS, ) + job4Run = DataProcessInstance.from_datajob( datajob=dataJob4, id=f"{jobFlow.id}-{uuid.uuid4()}" ) -job4Run.emit_process_start(emitter, int(datetime.utcnow().timestamp() * 1000)) +job4Run.emit_process_start(emitter, int(datetime.now(timezone.utc).timestamp() * 1000)) + job4Run.emit_process_end( - emitter, int(datetime.utcnow().timestamp() * 1000), result=InstanceRunResult.SUCCESS + emitter, + int(datetime.now(timezone.utc).timestamp() * 1000), + result=InstanceRunResult.SUCCESS, ) + jobFlowRun.emit_process_end( - emitter, int(datetime.utcnow().timestamp() * 1000), result=InstanceRunResult.SUCCESS + emitter, + int(datetime.now(timezone.utc).timestamp() * 1000), + result=InstanceRunResult.SUCCESS, ) diff --git a/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml index f5d57f36935ecc..377af98598d981 100644 --- a/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml +++ b/metadata-ingestion/examples/recipes/bigquery_to_datahub.dhub.yaml @@ -40,7 +40,7 @@ source: # - "schema.table.column" # deny: # - "*.*.*" - #lineage_client_project_id: project-id-1234567 + #storage_project_id: project-id-1234567 ## see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation sink: diff --git a/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml b/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml new file mode 100644 index 00000000000000..b1cfb6311ab17d --- /dev/null +++ b/metadata-ingestion/examples/recipes/csv_enricher_to_datahub_rest.dhub.yml @@ -0,0 +1,16 @@ +--- +# see https://datahubproject.io/docs/metadata-ingestion/source_docs/csv for complete documentation +source: + type: "csv-enricher" + config: + filename: "./examples/demo_data/csv_enricher_demo_data.csv" + write_semantics: "PATCH" + delimiter: "," + array_delimiter: "|" + + +# see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" \ No newline at end of file diff --git a/metadata-ingestion/examples/recipes/elasticsearch_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/elasticsearch_to_datahub.dhub.yaml index e3b56aa0e6c891..33876dff9efa0a 100644 --- a/metadata-ingestion/examples/recipes/elasticsearch_to_datahub.dhub.yaml +++ b/metadata-ingestion/examples/recipes/elasticsearch_to_datahub.dhub.yaml @@ -11,7 +11,11 @@ source: client_key: "./path/client.key" ssl_assert_hostname: False ssl_assert_fingerprint: "./path/cert.fingerprint" - + ingest_index_templates: False + # index_template_pattern: + # allow: + # - "^.+" + sink: type: "datahub-rest" config: diff --git a/metadata-ingestion/examples/transforms/custom_transform_example.py b/metadata-ingestion/examples/transforms/custom_transform_example.py index 85663d971092b5..57560e75cf7e92 100644 --- a/metadata-ingestion/examples/transforms/custom_transform_example.py +++ b/metadata-ingestion/examples/transforms/custom_transform_example.py @@ -61,13 +61,10 @@ def transform_aspect( # type: ignore assert aspect is None or isinstance(aspect, OwnershipClass) if owners_to_add: - ownership = ( - aspect - if aspect - else OwnershipClass( - owners=[], - ) + ownership = aspect or OwnershipClass( + owners=[], ) + ownership.owners.extend(owners_to_add) return ownership diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py index 05d8fb1c5804c6..d9b335fdc825c5 100644 --- a/metadata-ingestion/scripts/avro_codegen.py +++ b/metadata-ingestion/scripts/avro_codegen.py @@ -2,24 +2,20 @@ import types import unittest.mock from pathlib import Path -from typing import Dict, Iterable, List, Union +from typing import Any, Dict, Iterable, List, Union import avro.schema import click from avrogen import write_schema_files -def load_schema_file(schema_file: str) -> str: - with open(schema_file) as f: - raw_schema_text = f.read() +def load_schema_file(schema_file: Union[str, Path]) -> dict: + raw_schema_text = Path(schema_file).read_text() + return json.loads(raw_schema_text) - redo_spaces = json.dumps(json.loads(raw_schema_text), indent=2) - return redo_spaces - -def merge_schemas(schemas: List[str]) -> str: +def merge_schemas(schemas_obj: List[Any]) -> str: # Combine schemas. - schemas_obj = [json.loads(schema) for schema in schemas] merged = ["null"] + schemas_obj # Patch add_name method to NOT complain about duplicate names @@ -49,6 +45,7 @@ def default(self, obj): # This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py # Do not modify manually! +# pylint: skip-file # fmt: off """ autogen_footer = """ @@ -116,11 +113,85 @@ def make_load_schema_methods(schemas: Iterable[str]) -> str: ) +def annotate_aspects(aspects: List[dict], schema_class_file: Path) -> None: + schema_classes_lines = schema_class_file.read_text().splitlines() + line_lookup_table = {line: i for i, line in enumerate(schema_classes_lines)} + + # Create the Aspect class. + # We ensure that it cannot be instantiated directly, as + # per https://stackoverflow.com/a/7989101/5004662. + schema_classes_lines[ + line_lookup_table["__SCHEMAS: Dict[str, RecordSchema] = {}"] + ] += """ + +class _Aspect(DictWrapper): + ASPECT_NAME: str = None # type: ignore + + def __init__(self): + if type(self) is _Aspect: + raise TypeError("_Aspect is an abstract class, and cannot be instantiated directly.") + super().__init__() + + @classmethod + def get_aspect_name(cls) -> str: + return cls.ASPECT_NAME # type: ignore +""" + + for aspect in aspects: + className = f'{aspect["name"]}Class' + aspectName = aspect["Aspect"]["name"] + class_def_original = f"class {className}(DictWrapper):" + + # Make the aspects inherit from the Aspect class. + class_def_line = line_lookup_table[class_def_original] + schema_classes_lines[class_def_line] = f"class {className}(_Aspect):" + + # Define the ASPECT_NAME class attribute. + # There's always an empty line between the docstring and the RECORD_SCHEMA class attribute. + # We need to find it and insert our line of code there. + empty_line = class_def_line + 1 + while not ( + schema_classes_lines[empty_line].strip() == "" + and schema_classes_lines[empty_line + 1] + .strip() + .startswith("RECORD_SCHEMA = ") + ): + empty_line += 1 + schema_classes_lines[empty_line] = f"\n ASPECT_NAME = '{aspectName}'" + + schema_class_file.write_text("\n".join(schema_classes_lines)) + + @click.command() -@click.argument("schema_files", type=click.Path(exists=True), nargs=-1, required=True) +@click.argument( + "schemas_path", type=click.Path(exists=True, file_okay=False), required=True +) @click.argument("outdir", type=click.Path(), required=True) -def generate(schema_files: List[str], outdir: str) -> None: - schemas: Dict[str, str] = {} +def generate(schemas_path: str, outdir: str) -> None: + required_schema_files = { + "mxe/MetadataChangeEvent.avsc", + "mxe/MetadataChangeProposal.avsc", + "usage/UsageAggregation.avsc", + "mxe/MetadataChangeLog.avsc", + "mxe/PlatformEvent.avsc", + "platform/event/v1/EntityChangeEvent.avsc", + } + + # Find all the aspect schemas / other important schemas. + aspect_file_stems: List[str] = [] + schema_files: List[Path] = [] + for schema_file in Path(schemas_path).glob("**/*.avsc"): + relative_path = schema_file.relative_to(schemas_path).as_posix() + if relative_path in required_schema_files: + schema_files.append(schema_file) + required_schema_files.remove(relative_path) + elif load_schema_file(schema_file).get("Aspect"): + aspect_file_stems.append(schema_file.stem) + schema_files.append(schema_file) + + assert not required_schema_files, f"Schema files not found: {required_schema_files}" + + schemas: Dict[str, dict] = {} for schema_file in schema_files: schema = load_schema_file(schema_file) schemas[Path(schema_file).stem] = schema @@ -132,12 +203,18 @@ def generate(schema_files: List[str], outdir: str) -> None: # Schema files post-processing. (Path(outdir) / "__init__.py").write_text("# This file is intentionally empty.\n") add_avro_python3_warning(Path(outdir) / "schema_classes.py") + annotate_aspects( + [schemas[aspect_file_stem] for aspect_file_stem in aspect_file_stems], + Path(outdir) / "schema_classes.py", + ) # Save raw schema files in codegen as well. schema_save_dir = Path(outdir) / "schemas" schema_save_dir.mkdir() for schema_out_file, schema in schemas.items(): - (schema_save_dir / f"{schema_out_file}.avsc").write_text(schema) + (schema_save_dir / f"{schema_out_file}.avsc").write_text( + json.dumps(schema, indent=2) + ) # Add load_schema method. with open(schema_save_dir / "__init__.py", "a") as schema_dir_init: diff --git a/metadata-ingestion/scripts/codegen.sh b/metadata-ingestion/scripts/codegen.sh index 41e933376ff05c..5e41a0a2c9eabd 100755 --- a/metadata-ingestion/scripts/codegen.sh +++ b/metadata-ingestion/scripts/codegen.sh @@ -6,24 +6,6 @@ OUTDIR=./src/datahub/metadata # Note: this assumes that datahub has already been built with `./gradlew build`. DATAHUB_ROOT=.. SCHEMAS_ROOT="$DATAHUB_ROOT/metadata-events/mxe-schemas/src/renamed/avro/com/linkedin" -FILES="$SCHEMAS_ROOT/mxe/MetadataChangeEvent.avsc $SCHEMAS_ROOT/mxe/MetadataChangeProposal.avsc $SCHEMAS_ROOT/usage/UsageAggregation.avsc $SCHEMAS_ROOT/mxe/MetadataChangeLog.avsc $SCHEMAS_ROOT/mxe/PlatformEvent.avsc $SCHEMAS_ROOT/platform/event/v1/EntityChangeEvent.avsc" -# Since we depend on jq, check if jq is installed -if ! which jq > /dev/null; then - echo "jq is not installed. Please install jq and rerun (https://stedolan.github.io/jq/)" - exit 1 -fi -find $SCHEMAS_ROOT -name "*.avsc" | sort | while read file -do -# Add all other files that are aspects but not included in the above - if (jq '.Aspect' -e $file > /dev/null) - then - FILES="${FILES} ${file}" - fi - echo $FILES > /tmp/codegen_files.txt -done - -FILES=$(cat /tmp/codegen_files.txt) - -rm -r $OUTDIR || true -python scripts/avro_codegen.py $FILES $OUTDIR +rm -r $OUTDIR 2>/dev/null || true +python scripts/avro_codegen.py $SCHEMAS_ROOT $OUTDIR diff --git a/metadata-ingestion/scripts/datahub_preflight.sh b/metadata-ingestion/scripts/datahub_preflight.sh index debe12dbc52f5c..2450d8d287ca4a 100755 --- a/metadata-ingestion/scripts/datahub_preflight.sh +++ b/metadata-ingestion/scripts/datahub_preflight.sh @@ -1,11 +1,25 @@ #!/bin/bash -e +#From https://stackoverflow.com/questions/4023830/how-to-compare-two-strings-in-dot-separated-version-format-in-bash +verlte() { + [ "$1" == "$(echo -e "$1\n$2" | sort -V | head -n1)" ] +} + brew_install() { - printf '\n🔎 Checking if %s installed\n' "${1}" - if brew list "$1" &>/dev/null; then - printf '✅ %s is already installed\n' "${1}" + package=${1} + required_version=${2} + printf '\n🔎 Checking if %s installed\n' "${package}" + version=$(brew list --version|grep "$1"|awk '{ print $2 }') + + if [ -n "${version}" ]; then + if [ -n "$2" ] && ! verlte "${required_version}" "${version}"; then + printf '🔽 %s is installed but its version %s is lower than the required %s\n' "${package}" "${version}" "${required_version}. Updating version..." + brew update && brew upgrade "$1" && printf '✅ %s is installed\n' "${package}" + else + printf '✅ %s is already installed\n' "${package} with version ${version}" + fi else - brew install "$1" && printf '✅ %s is installed\n' "${1}" + brew install "$1" && printf '✅ %s is installed\n' "${package}" fi } @@ -32,7 +46,7 @@ arm64_darwin_preflight() { fi printf "✨ Setting up librdkafka prerequisities\n" - brew_install "librdkafka" + brew_install "librdkafka" "1.9.1" brew_install "openssl@1.1" brew install "postgresql" diff --git a/metadata-ingestion/scripts/docgen.py b/metadata-ingestion/scripts/docgen.py index 03434d291d3024..d8e6ab221dd786 100644 --- a/metadata-ingestion/scripts/docgen.py +++ b/metadata-ingestion/scripts/docgen.py @@ -114,8 +114,7 @@ def get_enum_description( description = ( description + "." if description - else "" + " Allowed symbols are " + ",".join(enum_symbols) - ) + else "") + " Allowed symbols are " + ", ".join(enum_symbols) return description @@ -136,9 +135,6 @@ def gen_md_table( default=str(field_dict.get("default", "None")), ) ) - # md_str.append( - # f"| {get_prefixed_name(field_prefix, None)} | Enum | {field_dict['type']} | one of {','.join(field_dict['enum'])} |\n" - # ) elif "properties" in field_dict: for field_name, value in field_dict["properties"].items(): @@ -207,7 +203,6 @@ def gen_md_table( "additionalProperties" in value and "$ref" in value["additionalProperties"] ): - # breakpoint() value_ref = value["additionalProperties"]["$ref"] def_dict = get_definition_dict_from_definition( definitions_dict, value_ref @@ -462,7 +457,6 @@ def generate( if extra_docs: for path in glob.glob(f"{extra_docs}/**/*[.md|.yaml|.yml]", recursive=True): - # breakpoint() m = re.search("/docs/sources/(.*)/(.*).md", path) if m: @@ -555,7 +549,6 @@ def generate( source_documentation[platform_id] = ( source_documentation.get(platform_id) or {} ) - # breakpoint() create_or_update( source_documentation, @@ -605,9 +598,10 @@ def generate( os.makedirs(config_dir, exist_ok=True) with open(f"{config_dir}/{plugin_name}_config.json", "w") as f: f.write(source_config_class.schema_json(indent=2)) - - create_or_update(source_documentation, - [platform_id, "plugins", plugin_name, "config_schema"], + + create_or_update( + source_documentation, + [platform_id, "plugins", plugin_name, "config_schema"], source_config_class.schema_json(indent=2) or "", ) @@ -649,7 +643,9 @@ def generate( with open(platform_doc_file, "w") as f: if "name" in platform_docs: - f.write(f"import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n") + f.write( + f"import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n" + ) f.write(f"# {platform_docs['name']}\n") if len(platform_docs["plugins"].keys()) > 1: # More than one plugin used to provide integration with this platform @@ -722,8 +718,10 @@ def generate( f.write("\n```\n") if "config" in plugin_docs: f.write("\n### Config Details\n") - f.write(""" - \n\n""") + f.write( + """ + \n\n""" + ) f.write( "Note that a `.` is used to denote nested fields in the YAML recipe.\n\n" ) @@ -733,7 +731,8 @@ def generate( for doc in plugin_docs["config"]: f.write(doc) f.write("\n\n\n") - f.write(f""" + f.write( + f""" The [JSONSchema](https://json-schema.org/) for this configuration is inlined below.\n\n @@ -741,7 +740,8 @@ def generate( {plugin_docs['config_schema']} ```\n\n -\n\n""") +\n\n""" + ) # insert custom plugin docs after config details f.write(plugin_docs.get("custom_docs", "")) if "classname" in plugin_docs: diff --git a/metadata-ingestion/setup.cfg b/metadata-ingestion/setup.cfg index 39f920067224cd..2467c61983e5d8 100644 --- a/metadata-ingestion/setup.cfg +++ b/metadata-ingestion/setup.cfg @@ -1,5 +1,5 @@ [flake8] -max-complexity = 15 +max-complexity = 20 ignore = # Ignore: line length issues, since black's formatter will take care of them. E501, @@ -54,9 +54,10 @@ disallow_untyped_defs = yes asyncio_mode = auto addopts = --cov=src --cov-report term-missing --cov-config setup.cfg --strict-markers markers = + slow_unit: marks tests to only run slow unit tests (deselect with '-m not slow_unit') integration: marks tests to only run in integration (deselect with '-m "not integration"') + integration_batch_1: mark tests to only run in batch 1 of integration tests. This is done mainly for parallelisation (deselect with '-m not integration_batch_1') slow_integration: marks tests that are too slow to even run in integration (deselect with '-m "not slow_integration"') - slow_hana: marks tests that are too slow to even run in integration (deselect with '-m "not slow_hana"') testpaths = tests/unit tests/integration diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 7d36b2faae1ff7..af6398aecf6004 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -61,15 +61,44 @@ def get_long_description(): "types-Deprecated", "humanfriendly", "packaging", + "aiohttp<4", } kafka_common = { + # The confluent_kafka package provides a number of pre-built wheels for + # various platforms and architectures. However, it does not provide wheels + # for arm64 (including M1 Macs) or aarch64 (Docker's linux/arm64). This has + # remained an open issue on the confluent_kafka project for a year: + # - https://github.com/confluentinc/confluent-kafka-python/issues/1182 + # - https://github.com/confluentinc/confluent-kafka-python/pull/1161 + # + # When a wheel is not available, we must build from source instead. + # Building from source requires librdkafka to be installed. + # Most platforms have an easy way to install librdkafka: + # - MacOS: `brew install librdkafka` gives latest, which is 1.9.x or newer. + # - Debian: `apt install librdkafka` gives 1.6.0 (https://packages.debian.org/bullseye/librdkafka-dev). + # - Ubuntu: `apt install librdkafka` gives 1.8.0 (https://launchpad.net/ubuntu/+source/librdkafka). + # + # Moreover, confluent_kafka 1.9.0 introduced a hard compatibility break, and + # requires librdkafka >=1.9.0. As such, installing confluent_kafka 1.9.x on + # most arm64 Linux machines will fail, since it will build from source but then + # fail because librdkafka is too old. Hence, we have added an extra requirement + # that requires confluent_kafka<1.9.0 on non-MacOS arm64/aarch64 machines, which + # should ideally allow the builds to succeed in default conditions. We still + # want to allow confluent_kafka >= 1.9.0 for M1 Macs, which is why we can't + # broadly restrict confluent_kafka to <1.9.0. + # + # Note that this is somewhat of a hack, since we don't actually require the + # older version of confluent_kafka on those machines. Additionally, we will + # need monitor the Debian/Ubuntu PPAs and modify this rule if they start to + # support librdkafka >= 1.9.0. + "confluent_kafka>=1.5.0", + 'confluent_kafka<1.9.0; platform_system != "Darwin" and (platform_machine == "aarch64" or platform_machine == "arm64")', # We currently require both Avro libraries. The codegen uses avro-python3 (above) # schema parsers at runtime for generating and reading JSON into Python objects. # At the same time, we use Kafka's AvroSerializer, which internally relies on # fastavro for serialization. We do not use confluent_kafka[avro], since it # is incompatible with its own dep on avro-python3. - "confluent_kafka>=1.5.0", "fastavro>=1.2.0", } @@ -108,6 +137,11 @@ def get_long_description(): "botocore!=1.23.0", } +path_spec_common = { + "parse>=1.19.0", + "wcmatch", +} + looker_common = { # Looker Python SDK "looker-sdk==22.2.1" @@ -122,6 +156,14 @@ def get_long_description(): "protobuf<=3.20.1", } +redshift_common = { + "sqlalchemy-redshift", + "psycopg2-binary", + "GeoAlchemy2", + "sqllineage==1.3.5", + *path_spec_common, +} + snowflake_common = { # Snowflake plugin utilizes sql common *sql_common, @@ -159,16 +201,18 @@ def get_long_description(): "azure-identity==1.10.0", } -s3_base = { - *data_lake_base, - "moto[s3]", - "wcmatch", +s3_base = {*data_lake_base, "moto[s3]", *path_spec_common} + +delta_lake = { + *s3_base, + "deltalake", } usage_common = { "sqlparse", } + # Note: for all of these, framework_common will be added. plugins: Dict[str, Set[str]] = { # Sink plugins. @@ -178,6 +222,10 @@ def get_long_description(): "airflow": { "apache-airflow >= 1.10.2", }, + "circuit-breaker": { + "gql>=3.3.0", + "gql[requests]>=3.3.0", + }, "great-expectations": sql_common | {"sqllineage==1.3.5"}, # Source plugins # PyAthena is pinned with exact version because we use private method in PyAthena @@ -197,7 +245,8 @@ def get_long_description(): "datahub-business-glossary": set(), "data-lake": {*data_lake_base, *data_lake_profiling}, "s3": {*s3_base, *data_lake_profiling}, - "dbt": {"requests"} | aws_common, + "delta-lake": {*data_lake_profiling, *delta_lake}, + "dbt": {"requests", "cached_property"} | aws_common, "druid": sql_common | {"pydruid>=0.6.2"}, # Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws # UnsupportedProductError @@ -245,17 +294,10 @@ def get_long_description(): | {"psycopg2-binary", "acryl-pyhive[hive]>=0.6.12", "pymysql>=1.0.2"}, "pulsar": {"requests"}, "redash": {"redash-toolbelt", "sql-metadata", "sqllineage==1.3.5"}, - "redshift": sql_common - | {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2", "sqllineage==1.3.5"}, - "redshift-usage": sql_common - | usage_common - | { - "sqlalchemy-redshift", - "psycopg2-binary", - "GeoAlchemy2", - "sqllineage==1.3.5", - }, + "redshift": sql_common | redshift_common, + "redshift-usage": sql_common | usage_common | redshift_common, "sagemaker": aws_common, + "salesforce": {"simple-salesforce"}, "snowflake": snowflake_common, "snowflake-usage": snowflake_common | usage_common @@ -315,7 +357,7 @@ def get_long_description(): "flake8>=3.8.3", "flake8-tidy-imports>=4.3.0", "isort>=5.7.0", - "mypy>=0.920", + "mypy>=0.950", # pydantic 1.8.2 is incompatible with mypy 0.910. # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910. "pydantic>=1.9.0", @@ -323,7 +365,6 @@ def get_long_description(): "pytest-asyncio>=0.16.0", "pytest-cov>=2.8.1", "pytest-docker>=0.10.3,<0.12", - "tox", "deepdiff", "requests-mock", "freezegun", @@ -362,6 +403,7 @@ def get_long_description(): "starburst-trino-usage", "powerbi", "vertica", + "salesforce" # airflow is added below ] for dependency in plugins[plugin] @@ -376,6 +418,7 @@ def get_long_description(): { dependency for plugin in [ + "delta-lake", "feast", "iceberg", "lookml", @@ -389,6 +432,7 @@ def get_long_description(): { dependency for plugin in [ + "delta-lake", "iceberg", "lookml", ] @@ -416,11 +460,13 @@ def get_long_description(): *list( dependency for plugin in [ + "circuit-breaker", "clickhouse", "druid", "feast-legacy", "hana", "hive", + "kafka-connect", "ldap", "mongodb", "mssql", @@ -428,7 +474,6 @@ def get_long_description(): "mariadb", "snowflake", "redash", - "kafka-connect", "vertica", ] for dependency in plugins[plugin] @@ -442,6 +487,7 @@ def get_long_description(): dependency for plugin in [ "athena", + "delta-lake", "feast", "iceberg", ] @@ -452,6 +498,7 @@ def get_long_description(): entry_points = { "console_scripts": ["datahub = datahub.entrypoints:main"], "datahub.ingestion.source.plugins": [ + "csv-enricher = datahub.ingestion.source.csv_enricher:CSVEnricherSource", "file = datahub.ingestion.source.file:GenericFileSource", "sqlalchemy = datahub.ingestion.source.sql.sql_generic:SQLAlchemyGenericSource", "athena = datahub.ingestion.source.sql.athena:AthenaSource", @@ -461,6 +508,7 @@ def get_long_description(): "clickhouse = datahub.ingestion.source.sql.clickhouse:ClickHouseSource", "clickhouse-usage = datahub.ingestion.source.usage.clickhouse_usage:ClickHouseUsageSource", "data-lake = datahub.ingestion.source.data_lake:DataLakeSource", + "delta-lake = datahub.ingestion.source.delta_lake:DeltaLakeSource", "s3 = datahub.ingestion.source.s3:S3Source", "dbt = datahub.ingestion.source.dbt:DBTSource", "druid = datahub.ingestion.source.sql.druid:DruidSource", @@ -505,6 +553,7 @@ def get_long_description(): "vertica = datahub.ingestion.source.sql.vertica:VerticaSource", "presto-on-hive = datahub.ingestion.source.sql.presto_on_hive:PrestoOnHiveSource", "pulsar = datahub.ingestion.source.pulsar:PulsarSource", + "salesforce = datahub.ingestion.source.salesforce:SalesforceSource", ], "datahub.ingestion.sink.plugins": [ "file = datahub.ingestion.sink.file:FileSink", diff --git a/metadata-ingestion/src/datahub/__init__.py b/metadata-ingestion/src/datahub/__init__.py index 9ffe977182f68e..c787de400f85b9 100644 --- a/metadata-ingestion/src/datahub/__init__.py +++ b/metadata-ingestion/src/datahub/__init__.py @@ -1,3 +1,6 @@ +import sys +import warnings + # Published at https://pypi.org/project/acryl-datahub/. __package_name__ = "acryl-datahub" __version__ = "0.0.0.dev0" @@ -11,3 +14,11 @@ def nice_version_name() -> str: if is_dev_mode(): return "unavailable (installed in develop mode)" return __version__ + + +if sys.version_info < (3, 7): + warnings.warn( + "DataHub will require Python 3.7 or newer in a future release. " + "Please upgrade your Python version to continue using DataHub.", + FutureWarning, + ) diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py b/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py new file mode 100644 index 00000000000000..4dcf40454736b9 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py @@ -0,0 +1,8 @@ +from datahub.api.circuit_breaker.assertion_circuit_breaker import ( + AssertionCircuitBreaker, + AssertionCircuitBreakerConfig, +) +from datahub.api.circuit_breaker.operation_circuit_breaker import ( + OperationCircuitBreaker, + OperationCircuitBreakerConfig, +) diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py b/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py new file mode 100644 index 00000000000000..67a5b3630a455f --- /dev/null +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py @@ -0,0 +1,137 @@ +import logging +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional + +from pydantic import Field + +from datahub.api.circuit_breaker.circuit_breaker import ( + AbstractCircuitBreaker, + CircuitBreakerConfig, +) +from datahub.api.graphql import Assertion, Operation + +logger: logging.Logger = logging.getLogger(__name__) + + +class AssertionCircuitBreakerConfig(CircuitBreakerConfig): + verify_after_last_update: bool = Field( + default=True, + description="Whether to check if assertion happened after the dataset was last updated.", + ) + time_delta: timedelta = Field( + default=(timedelta(days=1)), + description="In what timeframe should accept an assertion result if updated field is not available for the dataset", + ) + + +class AssertionCircuitBreaker(AbstractCircuitBreaker): + r""" + DataHub Assertion Circuit Breaker + + The circuit breaker checks if there are passing assertion on the Dataset. + """ + config: AssertionCircuitBreakerConfig + + def __init__(self, config: AssertionCircuitBreakerConfig): + super().__init__(config.datahub_host, config.datahub_token, config.timeout) + self.config = config + self.assertion_api = Assertion( + datahub_host=config.datahub_host, + datahub_token=config.datahub_token, + timeout=config.timeout, + ) + + def get_last_updated(self, urn: str) -> Optional[datetime]: + operation_api: Operation = Operation(transport=self.assertion_api.transport) + operations = operation_api.query_operations(urn=urn) + if not operations: + return None + else: + return datetime.fromtimestamp(operations[0]["lastUpdatedTimestamp"] / 1000) + + def _check_if_assertion_failed( + self, assertions: List[Dict[str, Any]], last_updated: Optional[datetime] = None + ) -> bool: + @dataclass + class AssertionResult: + time: int + state: str + run_event: Any + + # If last_updated is set we expect to have at least one successfull assertion + if not assertions and last_updated: + return True + + result: bool = True + assertion_last_states: Dict[str, AssertionResult] = {} + for assertion in assertions: + if "runEvents" in assertion and "runEvents" in assertion["runEvents"]: + for run_event in assertion["runEvents"]["runEvents"]: + assertion_time = run_event["timestampMillis"] + assertion_state = run_event["result"]["type"] + assertion_urn = run_event["assertionUrn"] + if ( + assertion_urn not in assertion_last_states + or assertion_last_states[assertion_urn].time < assertion_time + ): + assertion_last_states[assertion_urn] = AssertionResult( + time=assertion_time, + state=assertion_state, + run_event=run_event, + ) + + for assertion_urn, last_assertion in assertion_last_states.items(): + if last_assertion.state == "FAILURE": + logger.debug(f"Runevent: {last_assertion.run_event}") + logger.info( + f"Assertion {assertion_urn} is failed on dataset. Breaking the circuit" + ) + return True + elif last_assertion.state == "SUCCESS": + logger.info(f"Found successful assertion: {assertion_urn}") + result = False + if last_updated is not None: + last_run = datetime.fromtimestamp(last_assertion.time / 1000) + if last_updated > last_run: + logger.error( + f"Missing assertion run for {assertion_urn}. The dataset was updated on {last_updated} but the last assertion run was at {last_run}" + ) + return True + return result + + def is_circuit_breaker_active(self, urn: str) -> bool: + r""" + Checks if the circuit breaker is active + + :param urn: The DataHub dataset unique identifier. + """ + + last_updated: Optional[datetime] = None + + if self.config.verify_after_last_update: + last_updated = self.get_last_updated(urn) + logger.info( + f"The dataset {urn} was last updated at {last_updated}, using this as min assertion date." + ) + + if not last_updated: + last_updated = datetime.now() - self.config.time_delta + logger.info( + f"Dataset {urn} doesn't have last updated or check_last_assertion_time is false, using calculated min assertion date {last_updated}" + ) + + assertions = self.assertion_api.query_assertion( + urn, + start_time_millis=int(last_updated.timestamp() * 1000), + status="COMPLETE", + ) + + if self._check_if_assertion_failed( + assertions, + last_updated if self.config.verify_after_last_update is True else None, + ): + logger.info(f"Dataset {urn} has failed or missing assertion(s).") + return True + + return False diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py b/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py new file mode 100644 index 00000000000000..f8554334281d85 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py @@ -0,0 +1,50 @@ +import logging +from abc import abstractmethod +from typing import Optional + +from gql import Client +from gql.transport.requests import RequestsHTTPTransport +from pydantic import Field + +from datahub.configuration import ConfigModel + +logger = logging.getLogger(__name__) + + +class CircuitBreakerConfig(ConfigModel): + datahub_host: str = Field(description="Url of the DataHub instance") + datahub_token: Optional[str] = Field(default=None, description="The datahub token") + timeout: Optional[int] = Field( + default=None, + description="The number of seconds to wait for your client to establish a connection to a remote machine", + ) + + +class AbstractCircuitBreaker: + client: Client + + def __init__( + self, + datahub_host: str, + datahub_token: Optional[str] = None, + timeout: Optional[int] = None, + ): + # logging.basicConfig(level=logging.DEBUG) + + # Select your transport with a defined url endpoint + self.transport = RequestsHTTPTransport( + url=datahub_host + "/api/graphql", + headers={"Authorization": "Bearer " + datahub_token} + if datahub_token is not None + else None, + method="POST", + timeout=timeout, + ) + self.client = Client( + transport=self.transport, + fetch_schema_from_transport=True, + ) + + @abstractmethod + def is_circuit_breaker_active(self, urn: str) -> bool: + pass diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/operation_circuit_breaker.py b/metadata-ingestion/src/datahub/api/circuit_breaker/operation_circuit_breaker.py new file mode 100644 index 00000000000000..58a4ee37d959b6 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/operation_circuit_breaker.py @@ -0,0 +1,81 @@ +import logging +from datetime import datetime, timedelta +from typing import Optional + +from pydantic import Field + +from datahub.api.circuit_breaker.circuit_breaker import ( + AbstractCircuitBreaker, + CircuitBreakerConfig, +) +from datahub.api.graphql import Operation + +logger: logging.Logger = logging.getLogger(__name__) + + +class OperationCircuitBreakerConfig(CircuitBreakerConfig): + time_delta: timedelta = Field( + default=(timedelta(days=1)), + description="In what timeframe should accept an operation result if updated field is not available for the dataset", + ) + + +class OperationCircuitBreaker(AbstractCircuitBreaker): + r""" + DataHub Operation Circuit Breaker + + The circuit breaker checks if there is an operation metadata for the dataset. + If there is no valid Operation metadata then the circuit breaker fails. + """ + + config: OperationCircuitBreakerConfig + operation_api: Operation + + def __init__(self, config: OperationCircuitBreakerConfig): + super().__init__(config.datahub_host, config.datahub_token, config.timeout) + self.config = config + self.operation_api = Operation( + datahub_host=config.datahub_host, + datahub_token=config.datahub_token, + timeout=config.timeout, + ) + + def is_circuit_breaker_active( + self, + urn: str, + partition: Optional[str] = None, + source_type: Optional[str] = None, + operation_type: Optional[str] = None, + ) -> bool: + r""" + Checks if the circuit breaker is active + + :param urn: The Datahub dataset unique identifier. + :param datahub_rest_conn_id: The REST DataHub connection id to communicate with DataHub + which is set as Airflow connection. + :param partition: The partition to check the operation. + :param source_type: The source type to filter on. If not set it will accept any source type. + See valid types here: https://datahubproject.io/docs/graphql/enums#operationsourcetype + :param operation_type: The operation type to filter on. If not set it will accept any source type. + See valid types here: https://datahubproject.io/docs/graphql/enums/#operationtype + """ + + start_time_millis: int = int( + (datetime.now() - self.config.time_delta).timestamp() * 1000 + ) + operations = self.operation_api.query_operations( + urn, + start_time_millis=start_time_millis, + partition=partition, + source_type=source_type, + operation_type=operation_type, + ) + logger.info(f"Operations: {operations}") + for operation in operations: + if ( + operation.get("lastUpdatedTimestamp") + and operation["lastUpdatedTimestamp"] >= start_time_millis + ): + return False + + return True diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py index c0378d554d5d31..588e66f19a0ef1 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py @@ -142,7 +142,7 @@ def emit( """ Emit the DataFlow entity to Datahub - :param emitter: Datahub Emitter to emit the proccess event + :param emitter: Datahub Emitter to emit the process event :param callback: (Optional[Callable[[Exception, str], None]]) the callback method for KafkaEmitter if it is used """ for mcp in self.generate_mcp(): diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py index 329eca7d9cd44b..ec86ad80226312 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py @@ -52,7 +52,7 @@ class DataJob: properties Dict[str, str]: Custom properties to set for the DataProcessInstance url (Optional[str]): Url which points to the DataJob at the orchestrator inlets (List[str]): List of urns the DataProcessInstance consumes - outlest (List[str]): List of urns the DataProcessInstance produces + outlets (List[str]): List of urns the DataProcessInstance produces input_datajob_urns: List[DataJobUrn] = field(default_factory=list) """ @@ -179,7 +179,7 @@ def emit( """ Emit the DataJob entity to Datahub - :param emitter: Datahub Emitter to emit the proccess event + :param emitter: Datahub Emitter to emit the process event :param callback: (Optional[Callable[[Exception, str], None]]) the callback method for KafkaEmitter if it is used :rtype: None """ diff --git a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py index 859e5700a51c3b..9b107d701ab02d 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py +++ b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py @@ -55,7 +55,7 @@ class DataProcessInstance: template_urn (Optional[Union[DataJobUrn, DataFlowUrn]]): The parent DataJob or DataFlow which was instantiated if applicable parent_instance (Optional[DataProcessInstanceUrn]): The parent execution's urn if applicable properties Dict[str, str]: Custom properties to set for the DataProcessInstance - url (Optional[str]): Url which points to the exection at the orchestrator + url (Optional[str]): Url which points to the execution at the orchestrator inlets (List[str]): List of entities the DataProcessInstance consumes outlets (List[str]): List of entities the DataProcessInstance produces """ @@ -118,10 +118,10 @@ def emit_process_start( """ :rtype: None - :param emitter: Datahub Emitter to emit the proccess event + :param emitter: Datahub Emitter to emit the process event :param start_timestamp_millis: (int) the execution start time in milliseconds :param attempt: the number of attempt of the execution with the same execution id - :param emit_template: (bool) If it is set the template of the execution (datajob, datflow) will be emitted as well. + :param emit_template: (bool) If it is set the template of the execution (datajob, dataflow) will be emitted as well. :param callback: (Optional[Callable[[Exception, str], None]]) the callback method for KafkaEmitter if it is used """ if emit_template and self.template_urn is not None: @@ -312,8 +312,8 @@ def from_datajob( :param datajob: (DataJob) the datajob from generate the DataProcessInstance :param id: (str) the id for the DataProcessInstance - :param clone_inlets: (bool) wheather to clone datajob's inlets - :param clone_outlets: (bool) wheather to clone datajob's outlets + :param clone_inlets: (bool) whether to clone datajob's inlets + :param clone_outlets: (bool) whether to clone datajob's outlets :return: DataProcessInstance """ dpi: DataProcessInstance = DataProcessInstance( diff --git a/metadata-ingestion/src/datahub/api/graphql/__init__.py b/metadata-ingestion/src/datahub/api/graphql/__init__.py new file mode 100644 index 00000000000000..e8c8d22bbb93df --- /dev/null +++ b/metadata-ingestion/src/datahub/api/graphql/__init__.py @@ -0,0 +1,2 @@ +from datahub.api.graphql.assertion import Assertion +from datahub.api.graphql.operation import Operation diff --git a/metadata-ingestion/src/datahub/api/graphql/assertion.py b/metadata-ingestion/src/datahub/api/graphql/assertion.py new file mode 100644 index 00000000000000..eb342e830e9969 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/graphql/assertion.py @@ -0,0 +1,91 @@ +import logging +from typing import Any, Dict, List, Optional + +from gql import gql + +from datahub.api.graphql.base import BaseApi + +logger = logging.getLogger(__name__) + + +class Assertion(BaseApi): + + ASSERTION_QUERY = """ +query dataset($urn: String!, $start: Int, $count: Int, $status: AssertionRunStatus,$limit: Int, $startTimeMillis:Long, $endTimeMillis:Long, $filter:FilterInput) { + dataset(urn: $urn) { + assertions(start: $start, count: $count){ + __typename + total + assertions{ + __typename + runEvents(status: $status, limit: $limit, startTimeMillis: $startTimeMillis, endTimeMillis: $endTimeMillis, filter: $filter) { + total + failed + succeeded + runEvents { + __typename + timestampMillis + partitionSpec { + __typename + type + partition + timePartition { + startTimeMillis + durationMillis + } + } + result { + __typename + type + rowCount + missingCount + unexpectedCount + actualAggValue + externalUrl + } + assertionUrn + } + } + } + } + } +} +""" + + def query_assertion( + self, + urn: str, + status: Optional[str] = None, + start_time_millis: Optional[int] = None, + end_time_millis: Optional[int] = None, + limit: Optional[int] = None, + filter: Optional[Dict[str, Optional[str]]] = None, + ) -> List[Dict[Any, Any]]: + r""" + Query assertions for a dataset. + + :param urn: The DataHub dataset unique identifier. + :param status: The assertion status to filter for. Every status will be accepted if it is not set. + See valid status at https://datahubproject.io/docs/graphql/enums#assertionrunstatus + :param start_time_millis: The start time in milliseconds from the assertions will be queried. + :param end_time_millis: The end time in milliseconds until the assertions will be queried. + :param filter: Additional key value filters which will be applied as AND query + """ + + result = self.client.execute( + gql(Assertion.ASSERTION_QUERY), + variable_values={ + "urn": urn, + "filter": self.gen_filter(filter) if filter else None, + "limit": limit, + "status": status, + "startTimeMillis": start_time_millis, + "endTimeMillis": end_time_millis, + }, + ) + + assertions = [] + if "dataset" in result and "assertions" in result["dataset"]: + assertions = result["dataset"]["assertions"]["assertions"] + + return assertions diff --git a/metadata-ingestion/src/datahub/api/graphql/base.py b/metadata-ingestion/src/datahub/api/graphql/base.py new file mode 100644 index 00000000000000..3654bd38816996 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/graphql/base.py @@ -0,0 +1,52 @@ +from typing import Dict, List, Optional + +from gql import Client +from gql.transport.requests import RequestsHTTPTransport + + +class BaseApi: + client: Client + + def __init__( + self, + datahub_host: Optional[str] = None, + datahub_token: Optional[str] = None, + timeout: Optional[int] = None, + transport: Optional[RequestsHTTPTransport] = None, + ): + # logging.basicConfig(level=logging.DEBUG) + + if transport: + self.transport = transport + else: + assert datahub_host is not None + # Select your transport with a defined url endpoint + self.transport = RequestsHTTPTransport( + url=datahub_host + "/api/graphql", + headers={"Authorization": "Bearer " + datahub_token} + if datahub_token is not None + else None, + method="POST", + timeout=timeout, + ) + + self.client = Client( + transport=self.transport, + fetch_schema_from_transport=True, + ) + + def gen_filter( + self, filters: Dict[str, Optional[str]] + ) -> Optional[Dict[str, List[Dict[str, str]]]]: + filter_expression: Optional[Dict[str, List[Dict[str, str]]]] = None + if not filters: + return None + + filter = [] + for key, value in filters.items(): + if value is None: + continue + filter.append({"field": key, "value": value}) + + filter_expression = {"and": filter} + return filter_expression diff --git a/metadata-ingestion/src/datahub/api/graphql/operation.py b/metadata-ingestion/src/datahub/api/graphql/operation.py new file mode 100644 index 00000000000000..5e1575e6f75dd2 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/graphql/operation.py @@ -0,0 +1,140 @@ +import logging +from typing import Any, Dict, List, Optional + +from gql import gql + +from datahub.api.graphql.base import BaseApi + +logger = logging.getLogger(__name__) + + +class Operation(BaseApi): + REPORT_OPERATION_MUTATION: str = """ +mutation reportOperation($urn: String!, $sourceType: OperationSourceType!, $operationType: OperationType!, $partition: String, $numAffectedRows: Long, $customProperties: [StringMapEntryInput!]) { + reportOperation(input: { + urn: $urn + sourceType: $sourceType + operationType: $operationType + partition: $partition + numAffectedRows: $numAffectedRows + customProperties: $customProperties + }) +}""" + + QUERY_OPERATIONS: str = """ + query dataset($urn: String!, $startTimeMillis: Long, $endTimeMillis: Long, $limit: Int, $filter:FilterInput) { + dataset(urn: $urn) { + urn + operations (startTimeMillis: $startTimeMillis, endTimeMillis: $endTimeMillis, limit: $limit, filter: $filter) { + __typename + actor + operationType + sourceType + numAffectedRows + partition + timestampMillis + lastUpdatedTimestamp + customProperties { + key + value + } + } + } +}""" + + def report_operation( + self, + urn: str, + source_type: str = "DATA_PROCESS", + operation_type: str = "UPDATE", + partition: Optional[str] = None, + num_affected_rows: int = 0, + custom_properties: Optional[Dict[str, str]] = None, + ) -> str: + r""" + Report operation metadata for a dataset. + :param source_type: The source type to filter on. If not set it will accept any source type. + Default value: DATA_PROCESS + See valid types here: https://datahubproject.io/docs/graphql/enums#operationsourcetype + :param operation_type: The operation type to filter on. If not set it will accept any source type. + Default value: "UPDATE" + See valid types here: https://datahubproject.io/docs/graphql/enums/#operationtype + :param partition: The partition to set the operation. + :param num_affected_rows: The number of rows affected by this operation. + :param custom_properties: Key/value pair of custom propertis + """ + variable_values = { + "urn": urn, + "sourceType": source_type, + "operationType": operation_type, + "numAffectedRows": num_affected_rows, + } + + if partition is not None: + variable_values["partition"] = partition + + if num_affected_rows is not None: + variable_values["numAffectedRows"] = num_affected_rows + + if custom_properties is not None: + variable_values["customProperties"] = custom_properties + + result = self.client.execute( + gql(Operation.REPORT_OPERATION_MUTATION), variable_values + ) + + return result["reportOperation"] + + def query_operations( + self, + urn: str, + start_time_millis: Optional[int] = None, + end_time_millis: Optional[int] = None, + limit: Optional[int] = None, + source_type: Optional[str] = None, + operation_type: Optional[str] = None, + partition: Optional[str] = None, + ) -> List[Dict[Any, Any]]: + r""" + Query operations for a dataset. + + :param urn: The DataHub dataset unique identifier. + :param start_time_millis: The start time in milliseconds from the operations will be queried. + :param end_time_millis: The end time in milliseconds until the operations will be queried. + :param limit: The maximum number of items to return. + :param source_type: The source type to filter on. If not set it will accept any source type. + See valid types here: https://datahubproject.io/docs/graphql/enums#operationsourcetype + :param operation_type: The operation type to filter on. If not set it will accept any source type. + See valid types here: https://datahubproject.io/docs/graphql/enums#operationsourcetype + :param partition: The partition to check the operation. + """ + + result = self.client.execute( + gql(Operation.QUERY_OPERATIONS), + variable_values={ + "urn": urn, + "startTimeMillis": start_time_millis, + "end_time_millis": end_time_millis, + "limit": limit, + "filter": self.gen_filter( + { + "sourceType": source_type, + "operationType": operation_type, + "partition": partition, + } + if filter + else None + ), + }, + ) + if "dataset" in result and "operations" in result["dataset"]: + operations = [] + if source_type is not None: + for operation in result["dataset"]["operations"]: + if operation["sourceType"] == source_type: + operations.append(operation) + else: + operations = result["dataset"]["operations"] + + return operations + return [] diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index ffd6da2b1c3399..59f78167ded323 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -10,13 +10,12 @@ import click import requests import yaml -from avrogen.dict_wrapper import DictWrapper from pydantic import BaseModel, ValidationError from requests.models import Response from requests.sessions import Session from datahub.emitter.mce_builder import Aspect -from datahub.emitter.rest_emitter import _make_curl_command +from datahub.emitter.request_helper import _make_curl_command from datahub.emitter.serialization_helper import post_json_transform from datahub.metadata.schema_classes import ( AssertionRunEventClass, @@ -54,6 +53,7 @@ SubTypesClass, UpstreamLineageClass, ViewPropertiesClass, + _Aspect, ) from datahub.utilities.urns.urn import Urn @@ -63,9 +63,16 @@ CONDENSED_DATAHUB_CONFIG_PATH = "~/.datahubenv" DATAHUB_CONFIG_PATH = os.path.expanduser(CONDENSED_DATAHUB_CONFIG_PATH) +DATAHUB_ROOT_FOLDER = os.path.expanduser("~/.datahub") + ENV_SKIP_CONFIG = "DATAHUB_SKIP_CONFIG" +ENV_METADATA_HOST_URL = "DATAHUB_GMS_URL" ENV_METADATA_HOST = "DATAHUB_GMS_HOST" +ENV_METADATA_PORT = "DATAHUB_GMS_PORT" +ENV_METADATA_PROTOCOL = "DATAHUB_GMS_PROTOCOL" ENV_METADATA_TOKEN = "DATAHUB_GMS_TOKEN" +ENV_DATAHUB_SYSTEM_CLIENT_ID = "DATAHUB_SYSTEM_CLIENT_ID" +ENV_DATAHUB_SYSTEM_CLIENT_SECRET = "DATAHUB_SYSTEM_CLIENT_SECRET" config_override: Dict = {} @@ -79,9 +86,9 @@ class DatahubConfig(BaseModel): gms: GmsConfig -def set_env_variables_override_config(host: str, token: Optional[str]) -> None: +def set_env_variables_override_config(url: str, token: Optional[str]) -> None: """Should be used to override the config when using rest emitter""" - config_override[ENV_METADATA_HOST] = host + config_override[ENV_METADATA_HOST_URL] = url if token is not None: config_override[ENV_METADATA_TOKEN] = token @@ -130,12 +137,26 @@ def get_details_from_config(): gms_token = gms_config.token return gms_host, gms_token except yaml.YAMLError as exc: - click.secho(f"{DATAHUB_CONFIG_PATH} malformatted, error: {exc}", bold=True) + click.secho(f"{DATAHUB_CONFIG_PATH} malformed, error: {exc}", bold=True) return None, None def get_details_from_env() -> Tuple[Optional[str], Optional[str]]: - return os.environ.get(ENV_METADATA_HOST), os.environ.get(ENV_METADATA_TOKEN) + host = os.environ.get(ENV_METADATA_HOST) + port = os.environ.get(ENV_METADATA_PORT) + token = os.environ.get(ENV_METADATA_TOKEN) + protocol = os.environ.get(ENV_METADATA_PROTOCOL, "http") + url = os.environ.get(ENV_METADATA_HOST_URL) + if port is not None: + url = f"{protocol}://{host}:{port}" + return url, token + # The reason for using host as URL is backward compatibility + # If port is not being used we assume someone is using host env var as URL + if url is None and host is not None: + log.warning( + f"Do not use {ENV_METADATA_HOST} as URL. Use {ENV_METADATA_HOST_URL} instead" + ) + return url or host, token def first_non_null(ls: List[Optional[str]]) -> Optional[str]: @@ -147,10 +168,18 @@ def guess_entity_type(urn: str) -> str: return urn.split(":")[2] -def get_host_and_token(): +def get_system_auth() -> Optional[str]: + system_client_id = os.environ.get(ENV_DATAHUB_SYSTEM_CLIENT_ID) + system_client_secret = os.environ.get(ENV_DATAHUB_SYSTEM_CLIENT_SECRET) + if system_client_id is not None and system_client_secret is not None: + return f"Basic {system_client_id}:{system_client_secret}" + return None + + +def get_url_and_token(): gms_host_env, gms_token_env = get_details_from_env() if len(config_override.keys()) > 0: - gms_host = config_override.get(ENV_METADATA_HOST) + gms_host = config_override.get(ENV_METADATA_HOST_URL) gms_token = config_override.get(ENV_METADATA_TOKEN) elif should_skip_config(): gms_host = gms_host_env @@ -164,17 +193,17 @@ def get_host_and_token(): def get_token(): - return get_host_and_token()[1] + return get_url_and_token()[1] def get_session_and_host(): session = requests.Session() - gms_host, gms_token = get_host_and_token() + gms_host, gms_token = get_url_and_token() if gms_host is None or gms_host.strip() == "": log.error( - f"GMS Host is not set. Use datahub init command or set {ENV_METADATA_HOST} env var" + f"GMS Host is not set. Use datahub init command or set {ENV_METADATA_HOST_URL} env var" ) return None, None @@ -194,13 +223,13 @@ def get_session_and_host(): def test_connection(): (session, host) = get_session_and_host() - url = host + "/config" + url = f"{host}/config" response = session.get(url) response.raise_for_status() def test_connectivity_complain_exit(operation_name: str) -> None: - """Test connectivty to metadata-service, log operation name and exit""" + """Test connectivity to metadata-service, log operation name and exit""" # First test connectivity try: test_connection() @@ -297,10 +326,7 @@ def post_delete_references_endpoint( path: str, cached_session_host: Optional[Tuple[Session, str]] = None, ) -> Tuple[int, List[Dict]]: - if not cached_session_host: - session, gms_host = get_session_and_host() - else: - session, gms_host = cached_session_host + session, gms_host = cached_session_host or get_session_and_host() url = gms_host + path payload = json.dumps(payload_obj) @@ -316,10 +342,7 @@ def post_delete_endpoint( path: str, cached_session_host: Optional[Tuple[Session, str]] = None, ) -> typing.Tuple[str, int]: - if not cached_session_host: - session, gms_host = get_session_and_host() - else: - session, gms_host = cached_session_host + session, gms_host = cached_session_host or get_session_and_host() url = gms_host + path return post_delete_endpoint_with_session_and_url(session, url, payload_obj) @@ -343,10 +366,11 @@ def post_delete_endpoint_with_session_and_url( def get_urns_by_filter( platform: Optional[str], - env: Optional[str], + env: Optional[str] = None, entity_type: str = "dataset", search_query: str = "*", include_removed: bool = False, + only_soft_deleted: Optional[bool] = None, ) -> Iterable[str]: session, gms_host = get_session_and_host() endpoint: str = "/entities?action=search" @@ -369,9 +393,7 @@ def get_urns_by_filter( "condition": "EQUAL", } ) - if platform is not None and ( - entity_type_lower == "chart" or entity_type_lower == "dashboard" - ): + if platform is not None and entity_type_lower in {"chart", "dashboard"}: filter_criteria.append( { "field": "tool", @@ -380,7 +402,15 @@ def get_urns_by_filter( } ) - if include_removed: + if only_soft_deleted: + filter_criteria.append( + { + "field": "removed", + "value": "true", + "condition": "EQUAL", + } + ) + elif include_removed: filter_criteria.append( { "field": "removed", @@ -479,10 +509,7 @@ def batch_get_ids( session, gms_host = get_session_and_host() endpoint: str = "/entitiesV2" url = gms_host + endpoint - ids_to_get = [] - for id in ids: - ids_to_get.append(Urn.url_encode(id)) - + ids_to_get = [Urn.url_encode(id) for id in ids] response = session.get( f"{url}?ids=List({','.join(ids_to_get)})", ) @@ -539,11 +566,7 @@ def get_entity( aspect: Optional[List] = None, cached_session_host: Optional[Tuple[Session, str]] = None, ) -> Dict: - if not cached_session_host: - session, gms_host = get_session_and_host() - else: - session, gms_host = cached_session_host - + session, gms_host = cached_session_host or get_session_and_host() if urn.startswith("urn%3A"): # we assume the urn is already encoded encoded_urn: str = urn @@ -556,7 +579,7 @@ def get_entity( endpoint: str = f"/entitiesV2/{encoded_urn}" if aspect and len(aspect): - endpoint = endpoint + "?aspects=List(" + ",".join(aspect) + ")" + endpoint = f"{endpoint}?aspects=List(" + ",".join(aspect) + ")" response = session.get(gms_host + endpoint) response.raise_for_status() @@ -569,12 +592,8 @@ def post_entity( aspect_name: str, aspect_value: Dict, cached_session_host: Optional[Tuple[Session, str]] = None, -) -> Dict: - if not cached_session_host: - session, gms_host = get_session_and_host() - else: - session, gms_host = cached_session_host - +) -> int: + session, gms_host = cached_session_host or get_session_and_host() endpoint: str = "/aspects/?action=ingestProposal" proposal = { @@ -671,10 +690,7 @@ def get_latest_timeseries_aspect_values( timeseries_aspect_name: str, cached_session_host: Optional[Tuple[Session, str]], ) -> Dict: - if not cached_session_host: - session, gms_host = get_session_and_host() - else: - session, gms_host = cached_session_host + session, gms_host = cached_session_host or get_session_and_host() query_body = { "urn": entity_urn, "entity": guess_entity_type(entity_urn), @@ -696,7 +712,7 @@ def get_aspects_for_entity( aspects: List[str] = [], typed: bool = False, cached_session_host: Optional[Tuple[Session, str]] = None, -) -> Dict[str, Union[dict, DictWrapper]]: +) -> Dict[str, Union[dict, _Aspect]]: # Process non-timeseries aspects non_timeseries_aspects: List[str] = [ a for a in aspects if a not in timeseries_class_to_aspect_name_map.values() @@ -725,16 +741,11 @@ def get_aspects_for_entity( aspect_value["aspect"]["value"] = json.loads( aspect_value["aspect"]["value"] ) - aspect_list.update( - # Follow the convention used for non-timeseries aspects. - { - aspect_cls.RECORD_SCHEMA.fullname.replace( - "pegasus2avro.", "" - ): aspect_value - } - ) + aspect_list[ + aspect_cls.RECORD_SCHEMA.fullname.replace("pegasus2avro.", "") + ] = aspect_value - aspect_map: Dict[str, Union[dict, DictWrapper]] = {} + aspect_map: Dict[str, Union[dict, _Aspect]] = {} for a in aspect_list.values(): aspect_name = a["name"] aspect_py_class: Optional[Type[Any]] = _get_pydantic_class_from_aspect_name( @@ -756,4 +767,4 @@ def get_aspects_for_entity( if aspects: return {k: v for (k, v) in aspect_map.items() if k in aspects} else: - return {k: v for (k, v) in aspect_map.items()} + return dict(aspect_map) diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py index 1ca0ac864693b1..a1ee1480314f16 100644 --- a/metadata-ingestion/src/datahub/cli/delete_cli.py +++ b/metadata-ingestion/src/datahub/cli/delete_cli.py @@ -94,7 +94,7 @@ def delete_for_registry( @click.option("--query", required=False, type=str) @click.option("--registry-id", required=False, type=str) @click.option("-n", "--dry-run", required=False, is_flag=True) -@click.option("--include-removed", required=False, is_flag=True) +@click.option("--only-soft-deleted", required=False, is_flag=True, default=False) @upgrade.check_upgrade @telemetry.with_telemetry def delete( @@ -107,7 +107,7 @@ def delete( query: str, registry_id: str, dry_run: bool, - include_removed: bool, + only_soft_deleted: bool, ) -> None: """Delete metadata from datahub using a single urn or a combination of filters""" @@ -118,6 +118,14 @@ def delete( "You must provide either an urn or a platform or an env or a query for me to delete anything" ) + include_removed: bool + if soft: + # For soft-delete include-removed does not make any sense + include_removed = False + else: + # For hard-delete we always include the soft-deleted items + include_removed = True + # default query is set to "*" if not provided query = "*" if query is None else query @@ -180,11 +188,6 @@ def delete( registry_id=registry_id, soft=soft, dry_run=dry_run ) else: - # log warn include_removed + hard is the only way to work - if include_removed and soft: - logger.warn( - "A filtered delete including soft deleted entities is redundant, because it is a soft delete by default. Please use --include-removed in conjunction with --hard" - ) # Filter based delete deletion_result = delete_with_filters( env=env, @@ -195,6 +198,7 @@ def delete( search_query=query, force=force, include_removed=include_removed, + only_soft_deleted=only_soft_deleted, ) if not dry_run: @@ -226,6 +230,7 @@ def delete_with_filters( entity_type: str = "dataset", env: Optional[str] = None, platform: Optional[str] = None, + only_soft_deleted: Optional[bool] = False, ) -> DeletionResult: session, gms_host = cli_utils.get_session_and_host() @@ -234,34 +239,80 @@ def delete_with_filters( logger.info(f"datahub configured with {gms_host}") emitter = rest_emitter.DatahubRestEmitter(gms_server=gms_host, token=token) batch_deletion_result = DeletionResult() - urns = [ - u - for u in cli_utils.get_urns_by_filter( - env=env, - platform=platform, - search_query=search_query, - entity_type=entity_type, - include_removed=include_removed, + + urns: List[str] = [] + if not only_soft_deleted: + urns = list( + cli_utils.get_urns_by_filter( + env=env, + platform=platform, + search_query=search_query, + entity_type=entity_type, + include_removed=False, + ) ) - ] + + soft_deleted_urns: List[str] = [] + if include_removed or only_soft_deleted: + soft_deleted_urns = list( + cli_utils.get_urns_by_filter( + env=env, + platform=platform, + search_query=search_query, + entity_type=entity_type, + only_soft_deleted=True, + ) + ) + + final_message = "" + if len(urns) > 0: + final_message = f"{len(urns)} " + if len(urns) > 0 and len(soft_deleted_urns) > 0: + final_message += "and " + if len(soft_deleted_urns) > 0: + final_message = f"{len(soft_deleted_urns)} (soft-deleted) " + logger.info( - f"Filter matched {len(urns)} entities. Sample: {choices(urns, k=min(5, len(urns)))}" + f"Filter matched {final_message} {entity_type} entities of {platform}. Sample: {choices(urns, k=min(5, len(urns)))}" ) - if not force: - click.confirm( - f"This will delete {len(urns)} entities. Are you sure?", abort=True + if len(urns) == 0 and len(soft_deleted_urns) == 0: + click.echo( + f"No urns to delete. Maybe you want to change entity_type={entity_type} or platform={platform} to be something different?" ) + return DeletionResult(end_time_millis=int(time.time() * 1000.0)) - for urn in progressbar.progressbar(urns, redirect_stdout=True): - one_result = _delete_one_urn( - urn, - soft=soft, - entity_type=entity_type, - dry_run=dry_run, - cached_session_host=(session, gms_host), - cached_emitter=emitter, + if not force and not dry_run: + type_delete = "soft" if soft else "permanently" + click.confirm( + f"This will {type_delete} delete {len(urns)} entities. Are you sure?", + abort=True, ) - batch_deletion_result.merge(one_result) + + if len(urns) > 0: + for urn in progressbar.progressbar(urns, redirect_stdout=True): + one_result = _delete_one_urn( + urn, + soft=soft, + entity_type=entity_type, + dry_run=dry_run, + cached_session_host=(session, gms_host), + cached_emitter=emitter, + ) + batch_deletion_result.merge(one_result) + + if len(soft_deleted_urns) > 0 and not soft: + click.echo("Starting to delete soft-deleted URNs") + for urn in progressbar.progressbar(soft_deleted_urns, redirect_stdout=True): + one_result = _delete_one_urn( + urn, + soft=soft, + entity_type=entity_type, + dry_run=dry_run, + cached_session_host=(session, gms_host), + cached_emitter=emitter, + is_soft_deleted=True, + ) + batch_deletion_result.merge(one_result) batch_deletion_result.end() return batch_deletion_result @@ -276,20 +327,25 @@ def _delete_one_urn( cached_emitter: Optional[rest_emitter.DatahubRestEmitter] = None, run_id: str = "delete-run-id", deletion_timestamp: int = _get_current_time(), + is_soft_deleted: Optional[bool] = None, ) -> DeletionResult: + soft_delete_msg: str = "" + if dry_run and is_soft_deleted: + soft_delete_msg = "(soft-deleted)" + deletion_result = DeletionResult() deletion_result.num_entities = 1 deletion_result.num_records = UNKNOWN_NUM_RECORDS # Default is unknown if soft: # Add removed aspect - if not cached_emitter: + if cached_emitter: + emitter = cached_emitter + else: _, gms_host = cli_utils.get_session_and_host() token = cli_utils.get_token() emitter = rest_emitter.DatahubRestEmitter(gms_server=gms_host, token=token) - else: - emitter = cached_emitter if not dry_run: emitter.emit_mcp( MetadataChangeProposalWrapper( @@ -305,18 +361,19 @@ def _delete_one_urn( ) else: logger.info(f"[Dry-run] Would soft-delete {urn}") + elif not dry_run: + payload_obj = {"urn": urn} + urn, rows_affected = cli_utils.post_delete_endpoint( + payload_obj, + "/entities?action=delete", + cached_session_host=cached_session_host, + ) + deletion_result.num_records = rows_affected else: - if not dry_run: - payload_obj = {"urn": urn} - urn, rows_affected = cli_utils.post_delete_endpoint( - payload_obj, - "/entities?action=delete", - cached_session_host=cached_session_host, - ) - deletion_result.num_records = rows_affected - else: - logger.info(f"[Dry-run] Would hard-delete {urn}") - deletion_result.num_records = UNKNOWN_NUM_RECORDS # since we don't know how many rows will be affected + logger.info(f"[Dry-run] Would hard-delete {urn} {soft_delete_msg}") + deletion_result.num_records = ( + UNKNOWN_NUM_RECORDS # since we don't know how many rows will be affected + ) deletion_result.end() return deletion_result diff --git a/metadata-ingestion/src/datahub/cli/docker.py b/metadata-ingestion/src/datahub/cli/docker.py index 3076e8bf90cbcd..d4d8fa3b4c39d1 100644 --- a/metadata-ingestion/src/datahub/cli/docker.py +++ b/metadata-ingestion/src/datahub/cli/docker.py @@ -8,11 +8,15 @@ import sys import tempfile import time +from pathlib import Path from typing import List, NoReturn, Optional import click +import pydantic import requests +from expandvars import expandvars +from datahub.cli.cli_utils import DATAHUB_ROOT_FOLDER from datahub.cli.docker_check import ( check_local_docker_containers, get_client_with_error, @@ -131,6 +135,218 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) -> return False +def _set_environment_variables( + version: Optional[str], + mysql_port: Optional[pydantic.PositiveInt], + zk_port: Optional[pydantic.PositiveInt], + kafka_broker_port: Optional[pydantic.PositiveInt], + schema_registry_port: Optional[pydantic.PositiveInt], + elastic_port: Optional[pydantic.PositiveInt], +) -> None: + if version is not None: + os.environ["DATAHUB_VERSION"] = version + if mysql_port is not None: + os.environ["DATAHUB_MAPPED_MYSQL_PORT"] = str(mysql_port) + + if zk_port is not None: + os.environ["DATAHUB_MAPPED_ZK_PORT"] = str(zk_port) + + if kafka_broker_port is not None: + os.environ["DATAHUB_MAPPED_KAFKA_BROKER_PORT"] = str(kafka_broker_port) + + if schema_registry_port is not None: + os.environ["DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT"] = str(schema_registry_port) + + if elastic_port is not None: + os.environ["DATAHUB_MAPPED_ELASTIC_PORT"] = str(elastic_port) + + +def _get_default_quickstart_compose_file() -> Optional[str]: + quickstart_folder = Path(DATAHUB_ROOT_FOLDER) / "quickstart" + try: + os.makedirs(quickstart_folder, exist_ok=True) + return f"{quickstart_folder}/docker-compose.yml" + except Exception as e: + logger.debug(f"Failed to identify a default quickstart compose file due to {e}") + return None + + +def _attempt_stop(quickstart_compose_file: List[pathlib.Path]) -> None: + default_quickstart_compose_file = _get_default_quickstart_compose_file() + compose_files_for_stopping = ( + quickstart_compose_file + if quickstart_compose_file + else [pathlib.Path(default_quickstart_compose_file)] + if default_quickstart_compose_file + else None + ) + if compose_files_for_stopping: + # docker-compose stop + base_command: List[str] = [ + "docker-compose", + *itertools.chain.from_iterable( + ("-f", f"{path}") for path in compose_files_for_stopping + ), + "-p", + "datahub", + ] + try: + logger.debug(f"Executing {base_command} stop") + subprocess.run( + [*base_command, "stop"], + check=True, + ) + click.secho("Stopped datahub successfully.", fg="green") + except subprocess.CalledProcessError: + click.secho( + "Error while stopping.", + fg="red", + ) + return + + +def _backup(backup_file: str) -> int: + resolved_backup_file = os.path.expanduser(backup_file) + dirname = os.path.dirname(resolved_backup_file) + logger.info(f"Creating directory {dirname} if it doesn't exist") + os.makedirs(dirname, exist_ok=True) + logger.info("Executing backup command") + result = subprocess.run( + [ + "bash", + "-c", + f"docker exec mysql mysqldump -u root -pdatahub datahub > {resolved_backup_file}", + ] + ) + logger.info( + f"Backup written to {resolved_backup_file} with status {result.returncode}" + ) + return result.returncode + + +def _restore( + restore_primary: bool, + restore_indices: Optional[bool], + primary_restore_file: Optional[str], +) -> int: + assert ( + restore_primary or restore_indices + ), "Either restore_primary or restore_indices must be set" + msg = "datahub> " + if restore_primary: + msg += f"Will restore primary database from {primary_restore_file}. " + if restore_indices is not False: + msg += ( + f"Will {'also ' if restore_primary else ''}re-build indexes from scratch. " + ) + else: + msg += "Will not re-build indexes. " + msg += "Press y to continue." + click.confirm(msg, abort=True) + if restore_primary: + assert primary_restore_file + resolved_restore_file = os.path.expanduser(primary_restore_file) + logger.info(f"Restoring primary db from backup at {resolved_restore_file}") + assert os.path.exists( + resolved_restore_file + ), f"File {resolved_restore_file} does not exist" + with open(resolved_restore_file, "r") as fp: + result = subprocess.run( + [ + "bash", + "-c", + "docker exec -i mysql bash -c 'mysql -uroot -pdatahub datahub '", + ], + stdin=fp, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if result.returncode != 0: + logger.error("Failed to run MySQL restore") + return result.returncode + else: + logger.info("Successfully restored primary backup.") + + # We run restore indices by default on primary restores, and also if the --restore-indices flag is explicitly set + if restore_indices is not False: + with tempfile.NamedTemporaryFile() as env_fp: + env_fp.write( + expandvars( + """ + # Required Environment Variables +EBEAN_DATASOURCE_USERNAME=datahub +EBEAN_DATASOURCE_PASSWORD=datahub +EBEAN_DATASOURCE_HOST=mysql:${DATAHUB_MAPPED_MYSQL_PORT:-3306} +EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:${DATAHUB_MAPPED_MYSQL_PORT:-3306}/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 +EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver +ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + +KAFKA_BOOTSTRAP_SERVER=broker:29092 +KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081} + +ELASTICSEARCH_HOST=elasticsearch +ELASTICSEARCH_PORT=${DATAHUB_MAPPED_ELASTIC_PORT:-9200} + +#NEO4J_HOST=http://:7474 +#NEO4J_URI=bolt:// +#NEO4J_USERNAME=neo4j +#NEO4J_PASSWORD=datahub + +DATAHUB_GMS_HOST=datahub-gms +DATAHUB_GMS_PORT=${DATAHUB_MAPPED_GMS_PORT:-8080} + +DATAHUB_MAE_CONSUMER_HOST=datahub-gms +DATAHUB_MAE_CONSUMER_PORT=9091 + +# Optional Arguments + +# Uncomment and set these to support SSL connection to Elasticsearch +# ELASTICSEARCH_USE_SSL= +# ELASTICSEARCH_SSL_PROTOCOL= +# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL= +# ELASTICSEARCH_SSL_TRUSTSTORE_FILE= +# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE= +# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD= +# ELASTICSEARCH_SSL_KEYSTORE_FILE= +# ELASTICSEARCH_SSL_KEYSTORE_TYPE= +# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= + """ + ).encode("utf-8") + ) + env_fp.flush() + if logger.isEnabledFor(logging.DEBUG): + with open(env_fp.name, "r") as env_fp_reader: + logger.debug(f"Env file contents: {env_fp_reader.read()}") + + # continue to issue the restore indices command + command = ( + "docker pull acryldata/datahub-upgrade:${DATAHUB_VERSION:-head}" + + f" && docker run --network datahub_network --env-file {env_fp.name} " + + "acryldata/datahub-upgrade:${DATAHUB_VERSION:-head} -u RestoreIndices -a clean" + ) + logger.info(f"Running index restore command: {command}") + result = subprocess.run( + args=[ + "bash", + "-c", + "docker pull acryldata/datahub-upgrade:" + + "${DATAHUB_VERSION:-head}" + + f" && docker run --network datahub_network --env-file {env_fp.name} " + + "acryldata/datahub-upgrade:${DATAHUB_VERSION:-head}" + + " -u RestoreIndices -a clean", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + logger.info( + f"Index restore command finished with status {result.returncode}" + ) + if result.returncode != 0: + logger.info(result.stderr) + logger.debug(result.stdout) + return result.returncode + + @docker.command() @click.option( "--version", @@ -146,6 +362,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) -> help="Attempt to build the containers locally before starting", ) @click.option( + "-f", "--quickstart-compose-file", type=click.Path(exists=True, dir_okay=False, readable=True), default=[], @@ -166,6 +383,91 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) -> default=None, help="If set, forces docker-compose to use that graph service implementation", ) +@click.option( + "--mysql-port", + type=pydantic.PositiveInt, + is_flag=False, + default=None, + help="If there is an existing mysql instance running on port 3306, set this to a free port to avoid port conflicts on startup", +) +@click.option( + "--zk-port", + type=pydantic.PositiveInt, + is_flag=False, + default=None, + help="If there is an existing zookeeper instance running on port 2181, set this to a free port to avoid port conflicts on startup", +) +@click.option( + "--kafka-broker-port", + type=pydantic.PositiveInt, + is_flag=False, + default=None, + help="If there is an existing Kafka broker running on port 9092, set this to a free port to avoid port conflicts on startup", +) +@click.option( + "--schema-registry-port", + type=pydantic.PositiveInt, + is_flag=False, + default=None, + help="If there is an existing process running on port 8081, set this to a free port to avoid port conflicts with Kafka schema registry on startup", +) +@click.option( + "--elastic-port", + type=pydantic.PositiveInt, + is_flag=False, + default=None, + help="If there is an existing Elasticsearch instance running on port 9092, set this to a free port to avoid port conflicts on startup", +) +@click.option( + "--stop", + type=bool, + is_flag=True, + default=False, + help="Use this flag to stop the running containers", +) +@click.option( + "--backup", + required=False, + is_flag=True, + default=False, + help="Run this to backup a running quickstart instance", +) +@click.option( + "--backup-file", + required=False, + type=click.Path(exists=False, dir_okay=False, readable=True, writable=True), + default=os.path.expanduser("~/.datahub/quickstart/backup.sql"), + show_default=True, + help="Run this to backup data from a running quickstart instance", +) +@click.option( + "--restore", + required=False, + is_flag=True, + default=False, + help="Run this to restore a running quickstart instance from a previous backup (see --backup)", +) +@click.option( + "--restore-file", + required=False, + type=str, + default=os.path.expanduser("~/.datahub/quickstart/backup.sql"), + help="Set this to provide a custom restore file", +) +@click.option( + "--restore-indices", + required=False, + is_flag=True, + default=False, + help="Enable the restoration of indices of a running quickstart instance. Note: Using --restore will automatically restore-indices unless you use the --no-restore-indices flag.", +) +@click.option( + "--no-restore-indices", + required=False, + is_flag=True, + default=False, + help="Disables the restoration of indices of a running quickstart instance when used in conjunction with --restore.", +) @upgrade.check_upgrade @telemetry.with_telemetry def quickstart( @@ -174,6 +476,18 @@ def quickstart( quickstart_compose_file: List[pathlib.Path], dump_logs_on_failure: bool, graph_service_impl: Optional[str], + mysql_port: Optional[pydantic.PositiveInt], + zk_port: Optional[pydantic.PositiveInt], + kafka_broker_port: Optional[pydantic.PositiveInt], + schema_registry_port: Optional[pydantic.PositiveInt], + elastic_port: Optional[pydantic.PositiveInt], + stop: bool, + backup: bool, + backup_file: str, + restore: bool, + restore_file: str, + restore_indices: bool, + no_restore_indices: bool, ) -> None: """Start an instance of DataHub locally using docker-compose. @@ -182,10 +496,28 @@ def quickstart( There are options to override the docker-compose config file, build the containers locally, and dump logs to the console or to a file if something goes wrong. """ + if backup: + _backup(backup_file) + return + if restore or restore_indices or no_restore_indices: + if not valid_restore_options(restore, restore_indices, no_restore_indices): + return + # execute restore + restore_indices_flag: Optional[bool] = None + if restore_indices: + restore_indices_flag = True + if no_restore_indices: + restore_indices_flag = False + _restore( + restore_primary=restore, + primary_restore_file=restore_file, + restore_indices=restore_indices_flag, + ) + return running_on_m1 = is_m1() if running_on_m1: - click.echo("Detected M1 machine") + click.secho("Detected M1 machine", fg="yellow") # Run pre-flight checks. issues = check_local_docker_containers(preflight_only=True) @@ -195,7 +527,16 @@ def quickstart( quickstart_compose_file = list( quickstart_compose_file ) # convert to list from tuple - if not quickstart_compose_file: + + auth_resources_folder = Path(DATAHUB_ROOT_FOLDER) / "plugins/auth/resources" + os.makedirs(auth_resources_folder, exist_ok=True) + + default_quickstart_compose_file = _get_default_quickstart_compose_file() + if stop: + _attempt_stop(quickstart_compose_file) + return + elif not quickstart_compose_file: + # download appropriate quickstart file should_use_neo4j = should_use_neo4j_for_graph_service(graph_service_impl) if should_use_neo4j and running_on_m1: click.secho( @@ -210,7 +551,11 @@ def quickstart( else GITHUB_M1_QUICKSTART_COMPOSE_URL ) - with tempfile.NamedTemporaryFile(suffix=".yml", delete=False) as tmp_file: + with open( + default_quickstart_compose_file, "wb" + ) if default_quickstart_compose_file else tempfile.NamedTemporaryFile( + suffix=".yml", delete=False + ) as tmp_file: path = pathlib.Path(tmp_file.name) quickstart_compose_file.append(path) click.echo(f"Fetching docker-compose file {github_file} from GitHub") @@ -221,8 +566,14 @@ def quickstart( logger.debug(f"Copied to {path}") # set version - if version is not None: - os.environ["DATAHUB_VERSION"] = version + _set_environment_variables( + version=version, + mysql_port=mysql_port, + zk_port=zk_port, + kafka_broker_port=kafka_broker_port, + schema_registry_port=schema_registry_port, + elastic_port=elastic_port, + ) base_command: List[str] = [ "docker-compose", @@ -321,14 +672,44 @@ def quickstart( ) +def valid_restore_options( + restore: bool, restore_indices: bool, no_restore_indices: bool +) -> bool: + if no_restore_indices and not restore: + click.secho( + "Using --no-restore-indices without a --restore isn't defined", fg="red" + ) + return False + if no_restore_indices and restore_indices: + click.secho( + "Using --restore-indices in conjunction with --no-restore-indices is undefined", + fg="red", + ) + return False + if restore and restore_indices: + click.secho( + "Using --restore automatically implies using --restore-indices, you don't need to set both. Continuing...", + fg="yellow", + ) + return True + return True + + @docker.command() @click.option( "--path", type=click.Path(exists=True, dir_okay=False), help=f"The MCE json file to ingest. Defaults to downloading {BOOTSTRAP_MCES_FILE} from GitHub", ) +@click.option( + "--token", + type=str, + is_flag=False, + default=None, + help="The token to be used when ingesting, used when datahub is deployed with METADATA_SERVICE_AUTH_ENABLED=true", +) @telemetry.with_telemetry -def ingest_sample_data(path: Optional[str]) -> None: +def ingest_sample_data(path: Optional[str], token: Optional[str]) -> None: """Ingest sample data into a running DataHub instance.""" if path is None: @@ -353,20 +734,23 @@ def ingest_sample_data(path: Optional[str]) -> None: # Run ingestion. click.echo("Starting ingestion...") - pipeline = Pipeline.create( - { - "source": { - "type": "file", - "config": { - "filename": path, - }, - }, - "sink": { - "type": "datahub-rest", - "config": {"server": "http://localhost:8080"}, + recipe: dict = { + "source": { + "type": "file", + "config": { + "filename": path, }, - } - ) + }, + "sink": { + "type": "datahub-rest", + "config": {"server": "http://localhost:8080"}, + }, + } + + if token is not None: + recipe["sink"]["config"]["token"] = token + + pipeline = Pipeline.create(recipe) pipeline.run() ret = pipeline.pretty_print_summary() sys.exit(ret) diff --git a/metadata-ingestion/src/datahub/cli/docker_check.py b/metadata-ingestion/src/datahub/cli/docker_check.py index e530f4d19616f3..25719cef2334d9 100644 --- a/metadata-ingestion/src/datahub/cli/docker_check.py +++ b/metadata-ingestion/src/datahub/cli/docker_check.py @@ -88,10 +88,11 @@ def check_local_docker_containers(preflight_only: bool = False) -> List[str]: if len(containers) == 0: issues.append("quickstart.sh or dev.sh is not running") else: - existing_containers = set(container.name for container in containers) + existing_containers = {container.name for container in containers} missing_containers = set(REQUIRED_CONTAINERS) - existing_containers - for missing in missing_containers: - issues.append(f"{missing} container is not present") + issues.extend( + f"{missing} container is not present" for missing in missing_containers + ) # Check that the containers are running and healthy. for container in containers: diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 55d3a0cdc04760..40577962720388 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -1,10 +1,13 @@ +import asyncio import csv +import functools import json import logging import os import pathlib import sys from datetime import datetime +from typing import Optional import click from click_default_group import DefaultGroup @@ -21,6 +24,7 @@ ) from datahub.configuration import SensitiveError from datahub.configuration.config_loader import load_config_file +from datahub.ingestion.run.connection import ConnectionManager from datahub.ingestion.run.pipeline import Pipeline from datahub.telemetry import telemetry from datahub.upgrade import upgrade @@ -79,10 +83,21 @@ def ingest() -> None: type=bool, is_flag=True, default=False, - help="Supress display of variable values in logs by supressing elaborae stacktrace (stackprinter) during ingestion failures", + help="Suppress display of variable values in logs by suppressing elaborate stacktrace (stackprinter) during ingestion failures", +) +@click.option( + "--test-source-connection", + type=bool, + is_flag=True, + default=False, + help="When set, ingestion will only test the source connection details from the recipe", +) +@click.option( + "--report-to", + type=str, + help="Provide an output file to produce a structured report from the run", ) @click.pass_context -@upgrade.check_upgrade @telemetry.with_telemetry @memory_leak_detector.with_leak_detection def run( @@ -93,17 +108,77 @@ def run( strict_warnings: bool, preview_workunits: int, suppress_error_logs: bool, + test_source_connection: bool, + report_to: str, ) -> None: """Ingest metadata into DataHub.""" + def run_pipeline_to_completion( + pipeline: Pipeline, structured_report: Optional[str] = None + ) -> int: + logger.info("Starting metadata ingestion") + try: + pipeline.run() + except Exception as e: + logger.info( + f"Source ({pipeline.config.source.type}) report:\n{pipeline.source.get_report().as_string()}" + ) + logger.info( + f"Sink ({pipeline.config.sink.type}) report:\n{pipeline.sink.get_report().as_string()}" + ) + # We dont want to log sensitive information in variables if the pipeline fails due to + # an unexpected error. Disable printing sensitive info to logs if ingestion is running + # with `--suppress-error-logs` flag. + if suppress_error_logs: + raise SensitiveError() from e + else: + raise e + else: + logger.info("Finished metadata ingestion") + pipeline.log_ingestion_stats() + pipeline.write_structured_report() + ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings) + return ret + + async def run_pipeline_async(pipeline: Pipeline) -> int: + loop = asyncio._get_running_loop() + return await loop.run_in_executor( + None, functools.partial(run_pipeline_to_completion, pipeline) + ) + + async def run_func_check_upgrade(pipeline: Pipeline) -> None: + version_stats_future = asyncio.ensure_future( + upgrade.retrieve_version_stats(pipeline.ctx.graph) + ) + the_one_future = asyncio.ensure_future(run_pipeline_async(pipeline)) + ret = await the_one_future + + # the one future has returned + if ret == 0: + try: + # we check the other futures quickly on success + version_stats = await asyncio.wait_for(version_stats_future, 0.5) + upgrade.maybe_print_upgrade_message(version_stats=version_stats) + except Exception as e: + logger.debug( + f"timed out with {e} waiting for version stats to be computed... skipping ahead." + ) + + sys.exit(ret) + + # main function begins logger.info("DataHub CLI version: %s", datahub_package.nice_version_name()) config_file = pathlib.Path(config) pipeline_config = load_config_file(config_file) + if test_source_connection: + _test_source_connection(report_to, pipeline_config) try: logger.debug(f"Using config: {pipeline_config}") - pipeline = Pipeline.create(pipeline_config, dry_run, preview, preview_workunits) + pipeline = Pipeline.create( + pipeline_config, dry_run, preview, preview_workunits, report_to + ) except ValidationError as e: click.echo(e, err=True) sys.exit(1) @@ -112,29 +187,24 @@ def run( # in a SensitiveError to prevent detailed variable-level information from being logged. raise SensitiveError() from e - logger.info("Starting metadata ingestion") + loop = asyncio.get_event_loop() + loop.run_until_complete(run_func_check_upgrade(pipeline)) + + +def _test_source_connection(report_to: Optional[str], pipeline_config: dict) -> None: try: - pipeline.run() + connection_report = ConnectionManager().test_source_connection(pipeline_config) + logger.info(connection_report.as_json()) + if report_to: + with open(report_to, "w") as out_fp: + out_fp.write(connection_report.as_json()) + logger.info(f"Wrote report successfully to {report_to}") + sys.exit(0) except Exception as e: - logger.info( - f"Source ({pipeline.config.source.type}) report:\n{pipeline.source.get_report().as_string()}" - ) - logger.info( - f"Sink ({pipeline.config.sink.type}) report:\n{pipeline.sink.get_report().as_string()}" - ) - # We dont want to log sensitive information in variables if the pipeline fails due to - # an unexpected error. Disable printing sensitive info to logs if ingestion is running - # with `--suppress-error-logs` flag. - if suppress_error_logs: - raise SensitiveError() from e - else: - raise e - else: - logger.info("Finished metadata pipeline") - pipeline.log_ingestion_stats() - ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings) - upgrade.maybe_print_upgrade_message(pipeline.ctx.graph) - sys.exit(ret) + logger.error(f"Failed to test connection due to {e}") + if connection_report: + logger.error(connection_report.as_json()) + sys.exit(1) def get_runs_url(gms_host: str) -> str: @@ -313,14 +383,14 @@ def rollback( current_time = now.strftime("%Y-%m-%d %H:%M:%S") try: - folder_name = report_dir + "/" + current_time + folder_name = f"{report_dir}/{current_time}" - ingestion_config_file_name = folder_name + "/config.json" + ingestion_config_file_name = f"{folder_name}/config.json" os.makedirs(os.path.dirname(ingestion_config_file_name), exist_ok=True) with open(ingestion_config_file_name, "w") as file_handle: json.dump({"run_id": run_id}, file_handle) - csv_file_name = folder_name + "/unsafe_entities.csv" + csv_file_name = f"{folder_name}/unsafe_entities.csv" with open(csv_file_name, "w") as file_handle: writer = csv.writer(file_handle) writer.writerow(["urn"]) @@ -329,4 +399,4 @@ def rollback( except IOError as e: print(e) - sys.exit("Unable to write reports to " + report_dir) + sys.exit(f"Unable to write reports to {report_dir}") diff --git a/metadata-ingestion/src/datahub/cli/migration_utils.py b/metadata-ingestion/src/datahub/cli/migration_utils.py index b383e2849b6b9b..e40e19dbd38ac6 100644 --- a/metadata-ingestion/src/datahub/cli/migration_utils.py +++ b/metadata-ingestion/src/datahub/cli/migration_utils.py @@ -5,6 +5,7 @@ from avrogen.dict_wrapper import DictWrapper from datahub.cli import cli_utils +from datahub.emitter.mce_builder import Aspect from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.metadata.schema_classes import ( ChangeTypeClass, @@ -212,14 +213,14 @@ def container_modifier( def modify_urn_list_for_aspect( aspect_name: str, - aspect: DictWrapper, + aspect: Aspect, relationship_type: str, old_urn: str, new_urn: str, -) -> DictWrapper: +) -> Aspect: - if hasattr(UrnListModifier, aspect_name + "_modifier"): - modifier = getattr(UrnListModifier, aspect_name + "_modifier") + if hasattr(UrnListModifier, f"{aspect_name}_modifier"): + modifier = getattr(UrnListModifier, f"{aspect_name}_modifier") return modifier( aspect=aspect, relationship_type=relationship_type, diff --git a/metadata-ingestion/src/datahub/cli/timeline_cli.py b/metadata-ingestion/src/datahub/cli/timeline_cli.py index 40c5af4e1e78a0..78988c6f165cf4 100644 --- a/metadata-ingestion/src/datahub/cli/timeline_cli.py +++ b/metadata-ingestion/src/datahub/cli/timeline_cli.py @@ -19,46 +19,40 @@ def pretty_field_path(field_path: str) -> str: - if field_path.startswith("[version=2.0]"): + if not field_path.startswith("[version=2.0]"): + return field_path # breakpoint() # parse schema field - tokens = [ - t - for t in field_path.split(".") - if not (t.startswith("[") or t.endswith("]")) - ] - path = ".".join(tokens) - return path - else: - return field_path + tokens = [ + t + for t in field_path.split(".") + if not t.startswith("[") and not t.endswith("]") + ] + + return ".".join(tokens) def pretty_id(id: Optional[str]) -> str: if not id: return "" - else: - # breakpoint() - assert id is not None - if id.startswith("urn:li:datasetField:") or id.startswith( - "urn:li:schemaField:" - ): - # parse schema field - schema_field_key = schema_field_urn_to_key( - id.replace("urn:li:datasetField", "urn:li:schemaField") - ) - if schema_field_key: - assert schema_field_key is not None - field_path = schema_field_key.fieldPath - - return f"{colored('field','cyan')}:{colored(pretty_field_path(field_path),'white')}" - if id.startswith("[version=2.0]"): - return f"{colored('field','cyan')}:{colored(pretty_field_path(id),'white')}" - - if id.startswith("urn:li:dataset"): - # parse dataset urn - dataset_key = dataset_urn_to_key(id) - if dataset_key: - return f"{colored('dataset','cyan')}:{colored(dataset_key.platform,'white')}:{colored(dataset_key.name,'white')}" + # breakpoint() + assert id is not None + if id.startswith("urn:li:datasetField:") or id.startswith("urn:li:schemaField:"): + schema_field_key = schema_field_urn_to_key( + id.replace("urn:li:datasetField", "urn:li:schemaField") + ) + if schema_field_key: + assert schema_field_key is not None + field_path = schema_field_key.fieldPath + + return f"{colored('field','cyan')}:{colored(pretty_field_path(field_path),'white')}" + if id.startswith("[version=2.0]"): + return f"{colored('field','cyan')}:{colored(pretty_field_path(id),'white')}" + + if id.startswith("urn:li:dataset"): + dataset_key = dataset_urn_to_key(id) + if dataset_key: + return f"{colored('dataset','cyan')}:{colored(dataset_key.platform,'white')}:{colored(dataset_key.name,'white')}" # failed to prettify, return original return id @@ -114,7 +108,7 @@ def get_timeline( required=True, multiple=True, type=str, - help="One of tag, glossary_term, technical_schema, documentation, ownership", + help="One of tag, glossary_term, technical_schema, documentation, owner", ) @click.option( "--start", @@ -148,7 +142,7 @@ def timeline( all_categories = [ "TAG", - "OWNERSHIP", + "OWNER", "GLOSSARY_TERM", "TECHNICAL_SCHEMA", "DOCUMENTATION", @@ -196,10 +190,10 @@ def timeline( ) change_color = ( "green" - if change_txn.get("semVerChange") == "MINOR" - or change_txn.get("semVerChange") == "PATCH" + if change_txn.get("semVerChange") in ["MINOR", "PATCH"] else "red" ) + print( f"{colored(change_instant,'cyan')} - {colored(change_txn['semVer'],change_color)}" ) diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 716572babc1e41..0fbaf1e74e3dd2 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -1,14 +1,32 @@ import re from abc import ABC, abstractmethod +from enum import Enum from typing import IO, Any, Dict, List, Optional, Pattern, cast -from pydantic import BaseModel +from pydantic import BaseModel, Extra, validator from pydantic.fields import Field class ConfigModel(BaseModel): class Config: - extra = "forbid" + extra = Extra.forbid + + +class TransformerSemantics(Enum): + """Describes semantics for aspect changes""" + + OVERWRITE = "OVERWRITE" # Apply changes blindly + PATCH = "PATCH" # Only apply differences from what exists already on the server + + +class TransformerSemanticsConfigModel(ConfigModel): + semantics: TransformerSemantics = TransformerSemantics.OVERWRITE + + @validator("semantics", pre=True) + def ensure_semantics_is_upper_case(cls, v: str) -> str: + if isinstance(v, str): + return v.upper() + return v class DynamicTypedConfig(ConfigModel): @@ -39,10 +57,7 @@ class OperationalError(PipelineExecutionError): def __init__(self, message: str, info: dict = None): self.message = message - if info: - self.info = info - else: - self.info = {} + self.info = info or {} class ConfigurationError(MetaError): @@ -108,11 +123,11 @@ class AllowDenyPattern(ConfigModel): allow: List[str] = Field( default=[".*"], - description="List of regex patterns for process groups to include in ingestion", + description="List of regex patterns to include in ingestion", ) deny: List[str] = Field( default=[], - description="List of regex patterns for process groups to exclude from ingestion.", + description="List of regex patterns to exclude from ingestion.", ) ignoreCase: Optional[bool] = Field( default=True, @@ -128,10 +143,7 @@ def alphabet_pattern(self) -> Pattern: @property def regex_flags(self) -> int: - if self.ignoreCase: - return re.IGNORECASE - else: - return 0 + return re.IGNORECASE if self.ignoreCase else 0 @classmethod def allow_all(cls) -> "AllowDenyPattern": @@ -142,11 +154,10 @@ def allowed(self, string: str) -> bool: if re.match(deny_pattern, string, self.regex_flags): return False - for allow_pattern in self.allow: - if re.match(allow_pattern, string, self.regex_flags): - return True - - return False + return any( + re.match(allow_pattern, string, self.regex_flags) + for allow_pattern in self.allow + ) def is_fully_specified_allow_list(self) -> bool: """ @@ -155,10 +166,9 @@ def is_fully_specified_allow_list(self) -> bool: pattern into a 'search for the ones that are allowed' pattern, which can be much more efficient in some cases. """ - for allow_pattern in self.allow: - if not self.alphabet_pattern.match(allow_pattern): - return False - return True + return all( + self.alphabet_pattern.match(allow_pattern) for allow_pattern in self.allow + ) def get_allowed_list(self) -> List[str]: """Return the list of allowed strings as a list, after taking into account deny patterns, if possible""" @@ -181,16 +191,12 @@ def all(cls) -> "KeyValuePattern": return KeyValuePattern() def value(self, string: str) -> List[str]: - for key in self.rules.keys(): - if re.match(key, string): - return self.rules[key] - return [] + return next( + (self.rules[key] for key in self.rules.keys() if re.match(key, string)), [] + ) def matched(self, string: str) -> bool: - for key in self.rules.keys(): - if re.match(key, string): - return True - return False + return any(re.match(key, string) for key in self.rules.keys()) def is_fully_specified_key(self) -> bool: """ @@ -199,10 +205,7 @@ def is_fully_specified_key(self) -> bool: pattern into a 'search for the ones that are allowed' pattern, which can be much more efficient in some cases. """ - for key in self.rules.keys(): - if not self.alphabet_pattern.match(key): - return True - return False + return any(not self.alphabet_pattern.match(key) for key in self.rules.keys()) def get(self) -> Dict[str, List[str]]: """Return the list of allowed strings as a list, after taking into account deny patterns, if possible""" diff --git a/metadata-ingestion/src/datahub/configuration/import_resolver.py b/metadata-ingestion/src/datahub/configuration/import_resolver.py index 56e232d0403241..19627c7b8c9569 100644 --- a/metadata-ingestion/src/datahub/configuration/import_resolver.py +++ b/metadata-ingestion/src/datahub/configuration/import_resolver.py @@ -8,9 +8,7 @@ def _pydantic_resolver(v: Union[T, str]) -> T: - if isinstance(v, str): - return import_path(v) - return v + return import_path(v) if isinstance(v, str) else v def pydantic_resolve_key(field: str) -> classmethod: diff --git a/metadata-ingestion/src/datahub/configuration/kafka.py b/metadata-ingestion/src/datahub/configuration/kafka.py index 876db21086e88e..197322a2e566e1 100644 --- a/metadata-ingestion/src/datahub/configuration/kafka.py +++ b/metadata-ingestion/src/datahub/configuration/kafka.py @@ -27,7 +27,7 @@ def bootstrap_host_colon_port_comma(cls, val: str) -> str: else: host = entry assert re.match( - # This regex is quite loose. Many invalid hostnames or IPs will slip through, + # This regex is quite loose. Many invalid hostname's or IPs will slip through, # but it serves as a good first line of validation. We defer to Kafka for the # remaining validation. r"^[\w\-\.\:]+$", diff --git a/metadata-ingestion/src/datahub/configuration/yaml.py b/metadata-ingestion/src/datahub/configuration/yaml.py index ee710b07bab3d2..1f1172836f7448 100644 --- a/metadata-ingestion/src/datahub/configuration/yaml.py +++ b/metadata-ingestion/src/datahub/configuration/yaml.py @@ -9,5 +9,4 @@ class YamlConfigurationMechanism(ConfigurationMechanism): """Ability to load configuration from yaml files""" def load_config(self, config_fp: IO) -> dict: - config = yaml.safe_load(config_fp) - return config + return yaml.safe_load(config_fp) diff --git a/metadata-ingestion/src/datahub/emitter/kafka_emitter.py b/metadata-ingestion/src/datahub/emitter/kafka_emitter.py index f2dc663cf0677a..001097a2e42f5b 100644 --- a/metadata-ingestion/src/datahub/emitter/kafka_emitter.py +++ b/metadata-ingestion/src/datahub/emitter/kafka_emitter.py @@ -49,12 +49,11 @@ def validate_topic_routes(cls: "KafkaEmitterConfig", values: dict) -> dict: raise ConfigurationError( "Using both topic and topic_routes configuration for Kafka is not supported. Use only topic_routes" ) - else: - logger.warning( - "Looks like you're using the deprecated `topic` configuration. Please migrate to `topic_routes`." - ) - # upgrade topic provided to topic_routes mce entry - values["topic_routes"][MCE_KEY] = values["topic"] + logger.warning( + "Looks like you're using the deprecated `topic` configuration. Please migrate to `topic_routes`." + ) + # upgrade topic provided to topic_routes mce entry + values["topic_routes"][MCE_KEY] = values["topic"] return values @@ -70,8 +69,7 @@ def __init__(self, config: KafkaEmitterConfig): def convert_mce_to_dict( mce: MetadataChangeEvent, ctx: SerializationContext ) -> dict: - tuple_encoding = mce.to_obj(tuples=True) - return tuple_encoding + return mce.to_obj(tuples=True) mce_avro_serializer = AvroSerializer( schema_str=getMetadataChangeEventSchema(), @@ -83,8 +81,7 @@ def convert_mcp_to_dict( mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper], ctx: SerializationContext, ) -> dict: - tuple_encoding = mcp.to_obj(tuples=True) - return tuple_encoding + return mcp.to_obj(tuples=True) mcp_avro_serializer = AvroSerializer( schema_str=getMetadataChangeProposalSchema(), diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index e6203933705cb1..3d8208f5088e50 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -9,7 +9,6 @@ from typing import Any, List, Optional, Type, TypeVar, Union, cast, get_type_hints import typing_inspect -from avrogen.dict_wrapper import DictWrapper from datahub.configuration.source_common import DEFAULT_ENV as DEFAULT_ENV_CONFIGURATION from datahub.emitter.serialization_helper import pre_json_transform @@ -33,9 +32,11 @@ UpstreamClass, UpstreamLineageClass, ) +from datahub.metadata.schema_classes import _Aspect as AspectAbstract from datahub.utilities.urns.dataset_urn import DatasetUrn logger = logging.getLogger(__name__) +Aspect = TypeVar("Aspect", bound=AspectAbstract) DEFAULT_ENV = DEFAULT_ENV_CONFIGURATION DEFAULT_FLOW_CLUSTER = "prod" @@ -104,8 +105,8 @@ def schema_field_urn_to_key(schema_field_urn: str) -> Optional[SchemaFieldKeyCla pattern = r"urn:li:schemaField:\((.*),(.*)\)" results = re.search(pattern, schema_field_urn) if results is not None: - dataset_urn: str = results.group(1) - field_path: str = results.group(2) + dataset_urn: str = results[1] + field_path: str = results[2] return SchemaFieldKeyClass(parent=dataset_urn, fieldPath=field_path) return None @@ -114,9 +115,7 @@ def dataset_urn_to_key(dataset_urn: str) -> Optional[DatasetKeyClass]: pattern = r"urn:li:dataset:\(urn:li:dataPlatform:(.*),(.*),(.*)\)" results = re.search(pattern, dataset_urn) if results is not None: - return DatasetKeyClass( - platform=results.group(1), name=results.group(2), origin=results.group(3) - ) + return DatasetKeyClass(platform=results[1], name=results[2], origin=results[3]) return None @@ -128,9 +127,7 @@ def container_new_urn_to_key(dataset_urn: str) -> Optional[ContainerKeyClass]: pattern = r"urn:dh:container:0:\((.*)\)" results = re.search(pattern, dataset_urn) if results is not None: - return ContainerKeyClass( - guid=results.group(1), - ) + return ContainerKeyClass(guid=results[1]) return None @@ -146,9 +143,7 @@ def container_urn_to_key(guid: str) -> Optional[ContainerKeyClass]: pattern = r"urn:li:container:(.*)" results = re.search(pattern, guid) if results is not None: - return ContainerKeyClass( - guid=results.group(1), - ) + return ContainerKeyClass(guid=results[1]) return None @@ -156,8 +151,7 @@ def datahub_guid(obj: dict) -> str: obj_str = json.dumps( pre_json_transform(obj), separators=(",", ":"), sort_keys=True ).encode("utf-8") - datahub_guid = md5(obj_str).hexdigest() - return datahub_guid + return md5(obj_str).hexdigest() def make_assertion_urn(assertion_id: str) -> str: @@ -223,7 +217,6 @@ def make_domain_urn(domain: str) -> str: def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str: - return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})" @@ -231,7 +224,6 @@ def make_ml_feature_urn( feature_table_name: str, feature_name: str, ) -> str: - return f"urn:li:mlFeature:({feature_table_name},{feature_name})" @@ -255,6 +247,10 @@ def make_ml_model_group_urn(platform: str, group_name: str, env: str) -> str: def is_valid_ownership_type(ownership_type: Optional[str]) -> bool: return ownership_type is not None and ownership_type in [ + OwnershipTypeClass.TECHNICAL_OWNER, + OwnershipTypeClass.BUSINESS_OWNER, + OwnershipTypeClass.DATA_STEWARD, + OwnershipTypeClass.NONE, OwnershipTypeClass.DEVELOPER, OwnershipTypeClass.DATAOWNER, OwnershipTypeClass.DELEGATE, @@ -295,10 +291,6 @@ def make_lineage_mce( return mce -# This bound isn't tight, but it's better than nothing. -Aspect = TypeVar("Aspect", bound=DictWrapper) - - def can_add_aspect(mce: MetadataChangeEventClass, AspectType: Type[Aspect]) -> bool: SnapshotType = type(mce.proposedSnapshot) @@ -364,7 +356,9 @@ def make_global_tag_aspect_with_tag_list(tags: List[str]) -> GlobalTagsClass: def make_ownership_aspect_from_urn_list( - owner_urns: List[str], source_type: Optional[Union[str, OwnershipSourceTypeClass]] + owner_urns: List[str], + source_type: Optional[Union[str, OwnershipSourceTypeClass]], + owner_type: Union[str, OwnershipTypeClass] = OwnershipTypeClass.DATAOWNER, ) -> OwnershipClass: for owner_urn in owner_urns: assert owner_urn.startswith("urn:li:corpuser:") or owner_urn.startswith( @@ -377,7 +371,7 @@ def make_ownership_aspect_from_urn_list( owners_list = [ OwnerClass( owner=owner_urn, - type=OwnershipTypeClass.DATAOWNER, + type=owner_type, source=ownership_source_type, ) for owner_urn in owner_urns diff --git a/metadata-ingestion/src/datahub/emitter/mcp.py b/metadata-ingestion/src/datahub/emitter/mcp.py index 1d410616fbc170..f80609b114f64f 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp.py +++ b/metadata-ingestion/src/datahub/emitter/mcp.py @@ -10,6 +10,7 @@ KafkaAuditHeaderClass, MetadataChangeProposalClass, SystemMetadataClass, + _Aspect, ) @@ -23,15 +24,32 @@ def _make_generic_aspect(codegen_obj: DictWrapper) -> GenericAspectClass: @dataclasses.dataclass class MetadataChangeProposalWrapper: + # TODO: remove manually aspectName from the codebase + # TODO: (after) remove aspectName field from this class + # TODO: infer entityType from entityUrn + # TODO: set changeType's default to UPSERT + entityType: str changeType: Union[str, ChangeTypeClass] entityUrn: Union[None, str] = None - entityKeyAspect: Union[None, DictWrapper] = None + entityKeyAspect: Union[None, _Aspect] = None auditHeader: Union[None, KafkaAuditHeaderClass] = None aspectName: Union[None, str] = None - aspect: Union[None, DictWrapper] = None + aspect: Union[None, _Aspect] = None systemMetadata: Union[None, SystemMetadataClass] = None + def __post_init__(self) -> None: + if not self.aspectName and self.aspect: + self.aspectName = self.aspect.get_aspect_name() + elif ( + self.aspectName + and self.aspect + and self.aspectName != self.aspect.get_aspect_name() + ): + raise ValueError( + f"aspectName {self.aspectName} does not match aspect type {type(self.aspect)} with name {self.aspect.get_aspect_name()}" + ) + def make_mcp(self) -> MetadataChangeProposalClass: serializedEntityKeyAspect: Union[None, GenericAspectClass] = None if isinstance(self.entityKeyAspect, DictWrapper): diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index 7aed2e29137492..ecb9146a21fd81 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -85,7 +85,6 @@ def add_domain_to_entity_wu( entityType=entity_type, changeType=ChangeTypeClass.UPSERT, entityUrn=f"{entity_urn}", - aspectName="domains", aspect=DomainsClass(domains=[domain_urn]), ) wu = MetadataWorkUnit(id=f"{domain_urn}-to-{entity_urn}", mcp=mcp) @@ -99,7 +98,6 @@ def add_owner_to_entity_wu( entityType=entity_type, changeType=ChangeTypeClass.UPSERT, entityUrn=f"{entity_urn}", - aspectName="ownership", aspect=OwnershipClass( owners=[ OwnerClass( @@ -120,7 +118,6 @@ def add_tags_to_entity_wu( entityType=entity_type, changeType=ChangeTypeClass.UPSERT, entityUrn=f"{entity_urn}", - aspectName="globalTags", aspect=GlobalTagsClass( tags=[TagAssociationClass(f"urn:li:tag:{tag}") for tag in tags] ), @@ -148,7 +145,6 @@ def gen_containers( changeType=ChangeTypeClass.UPSERT, entityUrn=f"{container_urn}", # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), - aspectName="containerProperties", aspect=ContainerProperties( name=name, description=description, @@ -164,7 +160,6 @@ def gen_containers( changeType=ChangeTypeClass.UPSERT, entityUrn=f"{container_urn}", # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), - aspectName="dataPlatformInstance", aspect=DataPlatformInstance( platform=f"{make_data_platform_urn(container_key.platform)}", ), @@ -180,7 +175,6 @@ def gen_containers( changeType=ChangeTypeClass.UPSERT, entityUrn=f"{container_urn}", # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), - aspectName="subTypes", aspect=SubTypesClass(typeNames=sub_types), ) wu = MetadataWorkUnit( @@ -220,7 +214,6 @@ def gen_containers( changeType=ChangeTypeClass.UPSERT, entityUrn=f"{container_urn}", # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), - aspectName="container", aspect=ContainerClass(container=parent_container_urn), # aspect=ContainerKeyClass(guid=database_container_key.guid()) ) @@ -233,7 +226,9 @@ def gen_containers( def add_dataset_to_container( - container_key: KeyType, dataset_urn: str + # FIXME: Union requires two or more type arguments + container_key: KeyType, + dataset_urn: str, ) -> Iterable[Union[MetadataWorkUnit]]: container_urn = make_container_urn( guid=container_key.guid(), @@ -243,7 +238,6 @@ def add_dataset_to_container( entityType="dataset", changeType=ChangeTypeClass.UPSERT, entityUrn=f"{dataset_urn}", - aspectName="container", aspect=ContainerClass(container=f"{container_urn}"), # aspect=ContainerKeyClass(guid=schema_container_key.guid()) ) @@ -261,7 +255,6 @@ def add_entity_to_container( entityType=entity_type, changeType=ChangeTypeClass.UPSERT, entityUrn=entity_urn, - aspectName="container", aspect=ContainerClass(container=f"{container_urn}"), ) wu = MetadataWorkUnit(id=f"container-{container_urn}-to-{entity_urn}", mcp=mcp) diff --git a/metadata-ingestion/src/datahub/emitter/request_helper.py b/metadata-ingestion/src/datahub/emitter/request_helper.py new file mode 100644 index 00000000000000..987aa732c8d38e --- /dev/null +++ b/metadata-ingestion/src/datahub/emitter/request_helper.py @@ -0,0 +1,22 @@ +import itertools +import shlex +from typing import List + +import requests + + +def _make_curl_command( + session: requests.Session, method: str, url: str, payload: str +) -> str: + fragments: List[str] = [ + "curl", + *itertools.chain( + *[ + ("-X", method), + *[("-H", f"{k!s}: {v!s}") for (k, v) in session.headers.items()], + ("--data", payload), + ] + ), + url, + ] + return " ".join(shlex.quote(fragment) for fragment in fragments) diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py index 810eb819b7374c..21abdd80d9c77b 100644 --- a/metadata-ingestion/src/datahub/emitter/rest_emitter.py +++ b/metadata-ingestion/src/datahub/emitter/rest_emitter.py @@ -1,8 +1,8 @@ import datetime -import itertools +import functools import json import logging -import shlex +import os from json.decoder import JSONDecodeError from typing import Any, Dict, List, Optional, Tuple, Union @@ -10,8 +10,10 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import HTTPError, RequestException +from datahub.cli.cli_utils import get_system_auth from datahub.configuration.common import ConfigurationError, OperationalError from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.request_helper import _make_curl_command from datahub.emitter.serialization_helper import pre_json_transform from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( MetadataChangeEvent, @@ -22,23 +24,6 @@ logger = logging.getLogger(__name__) -def _make_curl_command( - session: requests.Session, method: str, url: str, payload: str -) -> str: - fragments: List[str] = [ - "curl", - *itertools.chain( - *[ - ("-X", method), - *[("-H", f"{k!s}: {v!s}") for (k, v) in session.headers.items()], - ("--data", payload), - ] - ), - url, - ] - return " ".join(shlex.quote(fragment) for fragment in fragments) - - class DataHubRestEmitter: DEFAULT_CONNECT_TIMEOUT_SEC = 30 # 30 seconds should be plenty to connect DEFAULT_READ_TIMEOUT_SEC = ( @@ -51,7 +36,9 @@ class DataHubRestEmitter: 504, ] DEFAULT_RETRY_METHODS = ["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"] - DEFAULT_RETRY_MAX_TIMES = 3 + DEFAULT_RETRY_MAX_TIMES = int( + os.getenv("DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES", "3") + ) _gms_server: str _token: Optional[str] @@ -91,6 +78,10 @@ def __init__( ) if token: self._session.headers.update({"Authorization": f"Bearer {token}"}) + else: + system_auth = get_system_auth() + if system_auth is not None: + self._session.headers.update({"Authorization": system_auth}) if extra_headers: self._session.headers.update(extra_headers) @@ -143,6 +134,13 @@ def __init__( self._session.mount("http://", adapter) self._session.mount("https://", adapter) + # Shim session.request to apply default timeout values. + # Via https://stackoverflow.com/a/59317604. + self._session.request = functools.partial( # type: ignore + self._session.request, + timeout=(self._connect_timeout_sec, self._read_timeout_sec), + ) + def test_connection(self) -> dict: response = self._session.get(f"{self._gms_server}/config") if response.status_code == 200: @@ -239,12 +237,7 @@ def _emit_generic(self, url: str, payload: str) -> None: curl_command, ) try: - response = self._session.post( - url, - data=payload, - timeout=(self._connect_timeout_sec, self._read_timeout_sec), - ) - + response = self._session.post(url, data=payload) response.raise_for_status() except HTTPError as e: try: @@ -262,6 +255,16 @@ def _emit_generic(self, url: str, payload: str) -> None: "Unable to emit metadata to DataHub GMS", {"message": str(e)} ) from e + def __repr__(self) -> str: + token_str = ( + f" with token: {self._token[:4]}**********{self._token[-4:]}" + if self._token + else "" + ) + return ( + f"DataHubRestEmitter: configured to talk to {self._gms_server}{token_str}" + ) + class DatahubRestEmitter(DataHubRestEmitter): """This class exists as a pass-through for backwards compatibility""" diff --git a/metadata-ingestion/src/datahub/emitter/serialization_helper.py b/metadata-ingestion/src/datahub/emitter/serialization_helper.py index 5a348ce267b10f..cad4e9dd3270fc 100644 --- a/metadata-ingestion/src/datahub/emitter/serialization_helper.py +++ b/metadata-ingestion/src/datahub/emitter/serialization_helper.py @@ -16,10 +16,12 @@ def _json_transform(obj: Any, from_pattern: str, to_pattern: str) -> Any: field = obj["fieldDiscriminator"] return {field: _json_transform(obj[field], from_pattern, to_pattern)} - new_obj: Any = {} - for key, value in obj.items(): - if value is not None: - new_obj[key] = _json_transform(value, from_pattern, to_pattern) + new_obj: Any = { + key: _json_transform(value, from_pattern, to_pattern) + for key, value in obj.items() + if value is not None + } + return new_obj elif isinstance(obj, list): new_obj = [_json_transform(item, from_pattern, to_pattern) for item in obj] diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py index e286c0c2e542a4..eb57a97925ecfe 100644 --- a/metadata-ingestion/src/datahub/entrypoints.py +++ b/metadata-ingestion/src/datahub/entrypoints.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) # Configure some loggers. -logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.ERROR) logging.getLogger("snowflake").setLevel(level=logging.WARNING) # logging.getLogger("botocore").setLevel(logging.INFO) # logging.getLogger("google").setLevel(logging.INFO) @@ -164,20 +164,35 @@ def main(**kwargs): if isinstance(exc, (ConfigurationError, ValidationError)): logger.error(exc) else: - logger.error( + # only print stacktraces during debug + logger.debug( stackprinter.format( exc, line_wrap=MAX_CONTENT_WIDTH, truncate_vals=10 * MAX_CONTENT_WIDTH, + suppressed_vars=[ + r".*password.*", + r".*secret.*", + r".*key.*", + r".*access.*", + # needed because sometimes secrets are in url + r".*url.*", + # needed because sqlalchemy uses it underneath + # and passes all params + r".*cparams.*", + ], suppressed_paths=[r"lib/python.*/site-packages/click/"], **kwargs, ) ) + logger.error( + f"Command failed with {exc}. Run with --debug to get full trace" + ) logger.info( f"DataHub CLI version: {datahub_package.__version__} at {datahub_package.__file__}" ) - logger.info( + logger.debug( f"Python version: {sys.version} at {sys.executable} on {platform.platform()}" ) - logger.info(f"GMS config {get_gms_config()}") + logger.debug(f"GMS config {get_gms_config()}") sys.exit(1) diff --git a/metadata-ingestion/src/datahub/ingestion/api/committable.py b/metadata-ingestion/src/datahub/ingestion/api/committable.py index f1aada4477f1ab..e41eb24abc2d96 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/committable.py +++ b/metadata-ingestion/src/datahub/ingestion/api/committable.py @@ -55,7 +55,7 @@ def __init__( super(_CommittableConcrete, self).__init__(state_to_commit=state_to_commit) def has_successfully_committed(self) -> bool: - return True if not self.state_to_commit or self.committed else False + return bool(not self.state_to_commit or self.committed) @abstractmethod def get_previous_states( diff --git a/metadata-ingestion/src/datahub/ingestion/api/common.py b/metadata-ingestion/src/datahub/ingestion/api/common.py index 56c21a7f39c627..fd458f9b4fd980 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/common.py +++ b/metadata-ingestion/src/datahub/ingestion/api/common.py @@ -55,8 +55,8 @@ def __init__( self.pipeline_name = pipeline_name self.dry_run_mode = dry_run self.preview_mode = preview_mode - self.reporters: Dict[str, Committable] = dict() - self.checkpointers: Dict[str, Committable] = dict() + self.reporters: Dict[str, Committable] = {} + self.checkpointers: Dict[str, Committable] = {} self._set_dataset_urn_to_lower_if_needed() def _set_dataset_urn_to_lower_if_needed(self) -> None: @@ -81,11 +81,8 @@ def register_reporter(self, committable: Committable) -> None: self.reporters[committable.name] = committable def get_reporters(self) -> Iterable[Committable]: - for committable in self.reporters.values(): - yield committable + yield from self.reporters.values() def get_committables(self) -> Iterable[Tuple[str, Committable]]: - for reporting_item_commitable in self.reporters.items(): - yield reporting_item_commitable - for checkpointing_item_commitable in self.checkpointers.items(): - yield checkpointing_item_commitable + yield from self.reporters.items() + yield from self.checkpointers.items() diff --git a/metadata-ingestion/src/datahub/ingestion/api/decorators.py b/metadata-ingestion/src/datahub/ingestion/api/decorators.py index 9b3f35ae9d8116..8cf1cc3f0dca60 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/decorators.py +++ b/metadata-ingestion/src/datahub/ingestion/api/decorators.py @@ -3,7 +3,7 @@ from typing import Callable, Dict, Optional, Type from datahub.ingestion.api.common import PipelineContext -from datahub.ingestion.api.source import Source +from datahub.ingestion.api.source import Source, SourceCapability def config_class(config_cls: Type) -> Callable[[Type], Type]: @@ -37,8 +37,9 @@ def wrapper(cls: Type) -> Type: setattr( cls, "get_platform_id", - lambda: id if id else platform_name.lower().replace(" ", "-"), + lambda: id or platform_name.lower().replace(" ", "-"), ) + return cls if id and " " in id: @@ -56,7 +57,7 @@ class SupportStatus(Enum): """ INCUBATING = auto() """ - Incubating Sources are ready for DataHub Community adoption but have not been tested for a wide variety of edge-cases. We eagerly solicit feedback from the Community to streghten the connector; minor version changes may arise in future releases. + Incubating Sources are ready for DataHub Community adoption but have not been tested for a wide variety of edge-cases. We eagerly solicit feedback from the Community to strengthen the connector; minor version changes may arise in future releases. """ TESTING = auto() """ @@ -80,20 +81,6 @@ def wrapper(cls: Type) -> Type: return wrapper -class SourceCapability(Enum): - PLATFORM_INSTANCE = "Platform Instance" - DOMAINS = "Domains" - DATA_PROFILING = "Data Profiling" - USAGE_STATS = "Dataset Usage" - PARTITION_SUPPORT = "Partition Support" - DESCRIPTIONS = "Descriptions" - LINEAGE_COARSE = "Table-Level Lineage" - LINEAGE_FINE = "Column-level Lineage" - OWNERSHIP = "Extract Ownership" - DELETION_DETECTION = "Detect Deleted Entities" - TAGS = "Extract Tags" - - @dataclass class CapabilitySetting: capability: SourceCapability diff --git a/metadata-ingestion/src/datahub/ingestion/api/registry.py b/metadata-ingestion/src/datahub/ingestion/api/registry.py index f83921639c227e..a8529817e2500f 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/registry.py +++ b/metadata-ingestion/src/datahub/ingestion/api/registry.py @@ -79,16 +79,15 @@ def register_disabled( def _ensure_not_lazy(self, key: str) -> Union[Type[T], Exception]: path = self._mapping[key] - if isinstance(path, str): - try: - plugin_class = import_path(path) - self.register(key, plugin_class, override=True) - return plugin_class - except (AssertionError, ModuleNotFoundError, ImportError) as e: - self.register_disabled(key, e, override=True) - return e - else: + if not isinstance(path, str): return path + try: + plugin_class = import_path(path) + self.register(key, plugin_class, override=True) + return plugin_class + except (AssertionError, ImportError) as e: + self.register_disabled(key, e, override=True) + return e def is_enabled(self, key: str) -> bool: tp = self._mapping[key] diff --git a/metadata-ingestion/src/datahub/ingestion/api/report.py b/metadata-ingestion/src/datahub/ingestion/api/report.py index ad9834b83081b2..d2403301d85bf9 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/report.py +++ b/metadata-ingestion/src/datahub/ingestion/api/report.py @@ -2,7 +2,8 @@ import pprint import sys from dataclasses import dataclass -from typing import Dict +from enum import Enum +from typing import Any, Dict # The sort_dicts option was added in Python 3.8. if sys.version_info >= (3, 8): @@ -13,10 +14,36 @@ @dataclass class Report: + @staticmethod + def to_str(some_val: Any) -> str: + if isinstance(some_val, Enum): + return some_val.name + else: + return str(some_val) + + @staticmethod + def to_dict(some_val: Any) -> Any: + """A cheap way to generate a dictionary.""" + if hasattr(some_val, "as_obj"): + return some_val.as_obj() + if hasattr(some_val, "dict"): + return some_val.dict() + elif isinstance(some_val, list): + return [Report.to_dict(v) for v in some_val if v is not None] + elif isinstance(some_val, dict): + return { + Report.to_str(k): Report.to_dict(v) + for k, v in some_val.items() + if v is not None + } + else: + return Report.to_str(some_val) + def as_obj(self) -> dict: return { - key: value.as_obj() if hasattr(value, "as_obj") else value + str(key): Report.to_dict(value) for (key, value) in self.__dict__.items() + if value is not None # ignore nulls } def as_string(self) -> str: diff --git a/metadata-ingestion/src/datahub/ingestion/api/sink.py b/metadata-ingestion/src/datahub/ingestion/api/sink.py index 56030987bce5da..1fa961dd42836a 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/sink.py +++ b/metadata-ingestion/src/datahub/ingestion/api/sink.py @@ -109,3 +109,7 @@ def get_report(self) -> SinkReport: @abstractmethod def close(self) -> None: pass + + def configured(self) -> str: + """Override this method to output a human-readable and scrubbed version of the configured sink""" + return "" diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py index 2230e81585441b..45d594de3e5df9 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/source.py +++ b/metadata-ingestion/src/datahub/ingestion/api/source.py @@ -2,7 +2,10 @@ import sys from abc import ABCMeta, abstractmethod from dataclasses import dataclass, field -from typing import Dict, Generic, Iterable, List, TypeVar +from enum import Enum +from typing import Dict, Generic, Iterable, List, Optional, TypeVar, Union + +from pydantic import BaseModel import datahub from datahub.ingestion.api.closeable import Closeable @@ -10,6 +13,22 @@ from datahub.ingestion.api.report import Report +class SourceCapability(Enum): + PLATFORM_INSTANCE = "Platform Instance" + DOMAINS = "Domains" + DATA_PROFILING = "Data Profiling" + USAGE_STATS = "Dataset Usage" + PARTITION_SUPPORT = "Partition Support" + DESCRIPTIONS = "Descriptions" + LINEAGE_COARSE = "Table-Level Lineage" + LINEAGE_FINE = "Column-level Lineage" + OWNERSHIP = "Extract Ownership" + DELETION_DETECTION = "Detect Deleted Entities" + TAGS = "Extract Tags" + SCHEMA_METADATA = "Schema Metadata" + CONTAINERS = "Asset Containers" + + @dataclass class SourceReport(Report): workunits_produced: int = 0 @@ -38,6 +57,24 @@ def report_failure(self, key: str, reason: str) -> None: self.failures[key].append(reason) +class CapabilityReport(BaseModel): + """A report capturing the result of any capability evaluation""" + + capable: bool + failure_reason: Optional[str] = None + mitigation_message: Optional[str] = None + + +@dataclass +class TestConnectionReport(Report): + internal_failure: Optional[bool] = None + internal_failure_reason: Optional[str] = None + basic_connectivity: Optional[CapabilityReport] = None + capability_report: Optional[ + Dict[Union[SourceCapability, str], CapabilityReport] + ] = None + + WorkUnitType = TypeVar("WorkUnitType", bound=WorkUnit) @@ -67,3 +104,10 @@ def get_workunits(self) -> Iterable[WorkUnit]: @abstractmethod def get_report(self) -> SourceReport: pass + + +class TestableSource(Source): + @staticmethod + @abstractmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + raise NotImplementedError("This class does not implement this method") diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py index e5f976ff88dd56..51fdbd8fbdb680 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py @@ -365,11 +365,7 @@ def _schema_fields_from_dag( if generations and generations[0]: roots = generations[0] - leafs: List = [] - for node in graph: - if graph.out_degree(node) == 0: - leafs.append(node) - + leafs: List = [node for node in graph if graph.out_degree(node) == 0] type_of_nodes: Dict = nx.get_node_attributes(graph, "node_type") for root in roots: diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index e75ced1240a665..be4116ae00267c 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -1,7 +1,8 @@ import json import logging +import os from json.decoder import JSONDecodeError -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, Iterable, List, Optional, Type from avro.schema import RecordSchema from deprecated import deprecated @@ -14,6 +15,8 @@ from datahub.emitter.serialization_helper import post_json_transform from datahub.metadata.schema_classes import ( DatasetUsageStatisticsClass, + DomainPropertiesClass, + DomainsClass, GlobalTagsClass, GlossaryTermsClass, OwnershipClass, @@ -24,6 +27,11 @@ logger = logging.getLogger(__name__) +telemetry_enabled = ( + os.environ.get("DATAHUB_TELEMETRY_ENABLED", "true").lower() == "true" +) + + class DatahubClientConfig(ConfigModel): """Configuration class for holding connectivity to datahub gms""" @@ -51,6 +59,9 @@ def __init__(self, config: DatahubClientConfig) -> None: ca_certificate_path=self.config.ca_certificate_path, ) self.test_connection() + if not telemetry_enabled: + self.server_id = "missing" + return try: client_id: Optional[TelemetryClientIdClass] = self.get_aspect_v2( "urn:li:telemetry:clientId", TelemetryClientIdClass, "telemetryClientId" @@ -173,6 +184,13 @@ def get_ownership(self, entity_urn: str) -> Optional[OwnershipClass]: aspect_type=OwnershipClass, ) + def get_domain_properties(self, entity_urn: str) -> Optional[DomainPropertiesClass]: + return self.get_aspect_v2( + entity_urn=entity_urn, + aspect="domainProperties", + aspect_type=DomainPropertiesClass, + ) + def get_tags(self, entity_urn: str) -> Optional[GlobalTagsClass]: return self.get_aspect_v2( entity_urn=entity_urn, @@ -187,6 +205,13 @@ def get_glossary_terms(self, entity_urn: str) -> Optional[GlossaryTermsClass]: aspect_type=GlossaryTermsClass, ) + def get_domain(self, entity_urn: str) -> Optional[DomainsClass]: + return self.get_aspect_v2( + entity_urn=entity_urn, + aspect="domains", + aspect_type=DomainsClass, + ) + def get_usage_aspects_from_urn( self, entity_urn: str, start_timestamp: int, end_timestamp: int ) -> Optional[List[DatasetUsageStatisticsClass]]: @@ -278,3 +303,137 @@ def get_latest_timeseries_value( f"Failed to find {aspect_type} in response {aspect_json}" ) return None + + def get_aspects_for_entity( + self, + entity_urn: str, + aspects: List[str], + aspect_types: List[Type[Aspect]], + ) -> Optional[Dict[str, Optional[Aspect]]]: + """ + Get multiple aspects for an entity. To get a single aspect for an entity, use the `get_aspect_v2` method. + Warning: Do not use this method to determine if an entity exists! + This method will always return an entity, even if it doesn't exist. This is an issue with how DataHub server + responds to these calls, and will be fixed automatically when the server-side issue is fixed. + + :param str entity_urn: The urn of the entity + :param List[Type[Aspect]] aspect_type_list: List of aspect type classes being requested (e.g. [datahub.metadata.schema_classes.DatasetProperties]) + :param List[str] aspects_list: List of aspect names being requested (e.g. [schemaMetadata, datasetProperties]) + :return: Optionally, a map of aspect_name to aspect_value as a dictionary if present, aspect_value will be set to None if that aspect was not found. Returns None on HTTP status 404. + :rtype: Optional[Dict[str, Optional[Aspect]]] + :raises HttpError: if the HTTP response is not a 200 or a 404 + """ + assert len(aspects) == len( + aspect_types + ), f"number of aspects requested ({len(aspects)}) should be the same as number of aspect types provided ({len(aspect_types)})" + aspects_list = ",".join(aspects) + url: str = f"{self._gms_server}/entitiesV2/{Urn.url_encode(entity_urn)}?aspects=List({aspects_list})" + + response = self._session.get(url) + if response.status_code == 404: + # not found + return None + response.raise_for_status() + response_json = response.json() + + result: Dict[str, Optional[Aspect]] = {} + for aspect_type in aspect_types: + record_schema: RecordSchema = aspect_type.__getattribute__( + aspect_type, "RECORD_SCHEMA" + ) + if not record_schema: + logger.warning( + f"Failed to infer type name of the aspect from the aspect type class {aspect_type}. Continuing, but this will fail." + ) + else: + aspect_type_name = record_schema.props["Aspect"]["name"] + aspect_json = response_json.get("aspects", {}).get(aspect_type_name) + if aspect_json: + # need to apply a transform to the response to match rest.li and avro serialization + post_json_obj = post_json_transform(aspect_json) + result[aspect_type_name] = aspect_type.from_obj(post_json_obj["value"]) + else: + result[aspect_type_name] = None + + return result + + def _get_search_endpoint(self): + return f"{self.config.server}/entities?action=search" + + def get_domain_urn_by_name(self, domain_name: str) -> Optional[str]: + """Retrieve a domain urn based on its name. Returns None if there is no match found""" + + filters = [] + filter_criteria = [ + { + "field": "name", + "value": domain_name, + "condition": "EQUAL", + } + ] + + filters.append({"and": filter_criteria}) + search_body = { + "input": "*", + "entity": "domain", + "start": 0, + "count": 10, + "filter": {"or": filters}, + } + results: Dict = self._post_generic(self._get_search_endpoint(), search_body) + num_entities = results.get("value", {}).get("numEntities", 0) + if num_entities > 1: + logger.warning( + f"Got {num_entities} results for domain name {domain_name}. Will return the first match." + ) + entities_yielded: int = 0 + entities = [] + for x in results["value"]["entities"]: + entities_yielded += 1 + logger.debug(f"yielding {x['entity']}") + entities.append(x["entity"]) + return entities[0] if entities_yielded else None + + def get_container_urns_by_filter( + self, + env: Optional[str] = None, + search_query: str = "*", + ) -> Iterable[str]: + """Return container urns that match based on query""" + url = self._get_search_endpoint() + + container_filters = [] + for container_subtype in ["Database", "Schema", "Project", "Dataset"]: + filter_criteria = [] + + filter_criteria.append( + { + "field": "customProperties", + "value": f"instance={env}", + "condition": "EQUAL", + } + ) + + filter_criteria.append( + { + "field": "typeNames", + "value": container_subtype, + "condition": "EQUAL", + } + ) + container_filters.append({"and": filter_criteria}) + search_body = { + "input": search_query, + "entity": "container", + "start": 0, + "count": 10000, + "filter": {"or": container_filters}, + } + results: Dict = self._post_generic(url, search_body) + num_entities = results["value"]["numEntities"] + logger.debug(f"Matched {num_entities} containers") + entities_yielded: int = 0 + for x in results["value"]["entities"]: + entities_yielded += 1 + logger.debug(f"yielding {x['entity']}") + yield x["entity"] diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_reporting_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_reporting_provider.py index 568c41aac9dbd9..1bb89236cc51ad 100644 --- a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_reporting_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_reporting_provider.py @@ -115,7 +115,7 @@ def get_previous_states( ) -> List[ReportingJobStatesMap]: if not last_only: raise NotImplementedError( - "Currently supports retrieving only the last commited state." + "Currently supports retrieving only the last committed state." ) if filter_opt is not None: raise NotImplementedError( diff --git a/metadata-ingestion/src/datahub/ingestion/run/connection.py b/metadata-ingestion/src/datahub/ingestion/run/connection.py new file mode 100644 index 00000000000000..54b0ab9f22c65e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/run/connection.py @@ -0,0 +1,35 @@ +import logging + +from datahub import __version__ +from datahub.ingestion.api.source import TestableSource, TestConnectionReport +from datahub.ingestion.source.source_registry import source_registry + +logger = logging.getLogger(__name__) + + +class ConnectionManager: + """A class that helps build / manage / triage connections""" + + def test_source_connection(self, recipe_config_dict: dict) -> TestConnectionReport: + # pulls out the source component of the dictionary + # walks the type registry to find the source class + # calls specific class.test_connection + try: + source_type = recipe_config_dict.get("source", {}).get("type") + source_class = source_registry.get(source_type) + if ( + issubclass(source_class, TestableSource) + and source_class.test_connection != TestableSource.test_connection + ): + # validate that the class overrides the base implementation + return source_class.test_connection( + recipe_config_dict.get("source", {}).get("config", {}) + ) + else: + return TestConnectionReport( + internal_failure=True, + internal_failure_reason=f"Source {source_type} in library version {__version__} does not support test connection functionality.", + ) + except Exception as e: + logger.error(e) + raise e diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py index e26470f823a6c7..2a93c4ffe69e97 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py @@ -1,5 +1,6 @@ import datetime import itertools +import json import logging import uuid from typing import Any, Dict, Iterable, List, Optional @@ -7,6 +8,7 @@ import click from pydantic import root_validator, validator +from datahub.cli.cli_utils import get_url_and_token from datahub.configuration import config_loader from datahub.configuration.common import ( ConfigModel, @@ -53,11 +55,10 @@ def run_id_should_be_semantic( cls, v: Optional[str], values: Dict[str, Any], **kwargs: Any ) -> str: if v == "__DEFAULT_RUN_ID": - if "source" in values: - if hasattr(values["source"], "type"): - source_type = values["source"].type - current_time = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") - return f"{source_type}-{current_time}" + if "source" in values and hasattr(values["source"], "type"): + source_type = values["source"].type + current_time = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + return f"{source_type}-{current_time}" return str(uuid.uuid1()) # default run_id if we cannot infer a source type else: @@ -67,11 +68,12 @@ def run_id_should_be_semantic( @root_validator(pre=True) def default_sink_is_datahub_rest(cls, values: Dict[str, Any]) -> Any: if "sink" not in values: + gms_host, gms_token = get_url_and_token() default_sink_config = { "type": "datahub-rest", "config": { - "server": "${DATAHUB_GMS_HOST:-http://localhost:8080}", - "token": "${DATAHUB_GMS_TOKEN:-}", + "server": gms_host, + "token": gms_token, }, } # resolve env variables if present @@ -86,12 +88,11 @@ def default_sink_is_datahub_rest(cls, values: Dict[str, Any]) -> Any: def datahub_api_should_use_rest_sink_as_default( cls, v: Optional[DatahubClientConfig], values: Dict[str, Any], **kwargs: Any ) -> Optional[DatahubClientConfig]: - if v is None: - if "sink" in values and hasattr(values["sink"], "type"): - sink_type = values["sink"].type - if sink_type == "datahub-rest": - sink_config = values["sink"].config - v = DatahubClientConfig.parse_obj(sink_config) + if v is None and "sink" in values and hasattr(values["sink"], "type"): + sink_type = values["sink"].type + if sink_type == "datahub-rest": + sink_config = values["sink"].config + v = DatahubClientConfig.parse_obj(sink_config) return v @@ -120,17 +121,24 @@ class Pipeline: sink: Sink transformers: List[Transformer] + def _record_initialization_failure(self, e: Exception, msg: str) -> None: + self.pipeline_init_exception: Optional[Exception] = e + self.pipeline_init_failures: Optional[str] = f"{msg} due to {e}" + logger.exception(e) + def __init__( self, config: PipelineConfig, dry_run: bool = False, preview_mode: bool = False, preview_workunits: int = 10, + report_to: Optional[str] = None, ): self.config = config self.dry_run = dry_run self.preview_mode = preview_mode self.preview_workunits = preview_workunits + self.report_to = report_to self.ctx = PipelineContext( run_id=self.config.run_id, datahub_api=self.config.datahub_api, @@ -138,23 +146,59 @@ def __init__( dry_run=dry_run, preview_mode=preview_mode, ) + self.pipeline_init_failures = None + self.pipeline_init_exception = None sink_type = self.config.sink.type - sink_class = sink_registry.get(sink_type) - sink_config = self.config.sink.dict().get("config") or {} - self.sink: Sink = sink_class.create(sink_config, self.ctx) - logger.debug(f"Sink type:{self.config.sink.type},{sink_class} configured") - - source_type = self.config.source.type - source_class = source_registry.get(source_type) - self.source: Source = source_class.create( - self.config.source.dict().get("config", {}), self.ctx - ) - logger.debug(f"Source type:{source_type},{source_class} configured") + try: + sink_class = sink_registry.get(sink_type) + except Exception as e: + self._record_initialization_failure(e, "Failed to create a sink") + return + + try: + sink_config = self.config.sink.dict().get("config") or {} + self.sink: Sink = sink_class.create(sink_config, self.ctx) + logger.debug(f"Sink type:{self.config.sink.type},{sink_class} configured") + logger.info(f"Sink configured successfully. {self.sink.configured()}") + except Exception as e: + self._record_initialization_failure( + e, f"Failed to configure sink ({sink_type})" + ) + return - self.extractor_class = extractor_registry.get(self.config.source.extractor) + try: + source_type = self.config.source.type + source_class = source_registry.get(source_type) + except Exception as e: + self._record_initialization_failure(e, "Failed to create source") + return + + try: + self.source: Source = source_class.create( + self.config.source.dict().get("config", {}), self.ctx + ) + logger.debug(f"Source type:{source_type},{source_class} configured") + except Exception as e: + self._record_initialization_failure( + e, f"Failed to configure source ({source_type})" + ) + return + + try: + self.extractor_class = extractor_registry.get(self.config.source.extractor) + except Exception as e: + self._record_initialization_failure( + e, f"Failed to configure extractor ({self.config.source.extractor})" + ) + return + + try: + self._configure_transforms() + except ValueError as e: + self._record_initialization_failure(e, "Failed to configure transformers") + return - self._configure_transforms() self._configure_reporting() def _configure_transforms(self) -> None: @@ -197,6 +241,7 @@ def create( dry_run: bool = False, preview_mode: bool = False, preview_workunits: int = 10, + report_to: Optional[str] = None, ) -> "Pipeline": config = PipelineConfig.parse_obj(config_dict) return cls( @@ -204,11 +249,16 @@ def create( dry_run=dry_run, preview_mode=preview_mode, preview_workunits=preview_workunits, + report_to=report_to, ) def run(self) -> None: callback = LoggingCallback() + if self.pipeline_init_failures: + # no point continuing, return early + return + extractor: Extractor = self.extractor_class() for wu in itertools.islice( self.source.get_workunits(), @@ -224,8 +274,10 @@ def run(self) -> None: for record_envelope in self.transform(record_envelopes): if not self.dry_run: self.sink.write_record_async(record_envelope, callback) + except Exception as e: logger.error(f"Failed to extract some records due to: {e}") + extractor.close() if not self.dry_run: self.sink.handle_work_unit_end(wu) @@ -266,11 +318,10 @@ def process_commits(self) -> None: if self.source.get_report().failures or self.sink.get_report().failures else False ) - has_warnings: bool = ( - True - if self.source.get_report().warnings or self.sink.get_report().warnings - else False + has_warnings: bool = bool( + self.source.get_report().warnings or self.sink.get_report().warnings ) + for name, committable in self.ctx.get_committables(): commit_policy: CommitPolicy = committable.commit_policy @@ -297,6 +348,9 @@ def process_commits(self) -> None: logger.info(f"Successfully committed changes for {name}.") def raise_from_status(self, raise_warnings: bool = False) -> None: + if self.pipeline_init_exception: + raise self.pipeline_init_exception + if self.source.get_report().failures: raise PipelineExecutionError( "Source reported errors", self.source.get_report() @@ -311,32 +365,78 @@ def raise_from_status(self, raise_warnings: bool = False) -> None: ) def log_ingestion_stats(self) -> None: + if not self.pipeline_init_failures: + telemetry.telemetry_instance.ping( + "ingest_stats", + { + "source_type": self.config.source.type, + "sink_type": self.config.sink.type, + "records_written": stats.discretize( + self.sink.get_report().records_written + ), + }, + self.ctx.graph, + ) - telemetry.telemetry_instance.ping( - "ingest_stats", - { - "source_type": self.config.source.type, - "sink_type": self.config.sink.type, - "records_written": stats.discretize( - self.sink.get_report().records_written - ), - }, - self.ctx.graph, - ) + def _count_all_vals(self, d: Dict[str, List]) -> int: + result = 0 + for val in d.values(): + result += len(val) + return result def pretty_print_summary(self, warnings_as_failure: bool = False) -> int: click.echo() + if self.pipeline_init_failures: + click.secho(f"{self.pipeline_init_failures}", fg="red") + return 1 click.secho(f"Source ({self.config.source.type}) report:", bold=True) click.echo(self.source.get_report().as_string()) click.secho(f"Sink ({self.config.sink.type}) report:", bold=True) click.echo(self.sink.get_report().as_string()) click.echo() + workunits_produced = self.source.get_report().workunits_produced if self.source.get_report().failures or self.sink.get_report().failures: - click.secho("Pipeline finished with failures", fg="bright_red", bold=True) + num_failures_source = self._count_all_vals( + self.source.get_report().failures + ) + click.secho( + f"Pipeline finished with {num_failures_source} failures in source producing {workunits_produced} workunits", + fg="bright_red", + bold=True, + ) return 1 elif self.source.get_report().warnings or self.sink.get_report().warnings: - click.secho("Pipeline finished with warnings", fg="yellow", bold=True) + num_warn_source = self._count_all_vals(self.source.get_report().warnings) + click.secho( + f"Pipeline finished with {num_warn_source} warnings in source producing {workunits_produced} workunits", + fg="yellow", + bold=True, + ) return 1 if warnings_as_failure else 0 else: - click.secho("Pipeline finished successfully", fg="green", bold=True) + click.secho( + f"Pipeline finished successfully producing {workunits_produced} workunits", + fg="green", + bold=True, + ) return 0 + + def write_structured_report(self) -> None: + if not self.report_to: + return + report = { + "source": { + "type": self.config.source.type, + "report": self.source.get_report().as_obj(), + }, + "sink": { + "type": self.config.sink.type, + "report": self.sink.get_report().as_obj(), + }, + } + try: + with open(self.report_to, "w") as report_out: + json.dump(report, report_out) + logger.info(f"Wrote report successfully to {report_out}") + except Exception as e: + logger.error(f"Failed to write structured report due to {e}") diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py index f931b9039303a0..93d3aa5f6c85d3 100644 --- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_kafka.py @@ -77,8 +77,8 @@ def write_record_async( self.report, record_envelope, write_callback ).kafka_callback, ) - elif isinstance(record, MetadataChangeProposalWrapper) or isinstance( - record, MetadataChangeProposalClass + elif isinstance( + record, (MetadataChangeProposalWrapper, MetadataChangeProposalClass) ): self.emitter.emit_mcp_async( record, diff --git a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py index 74e536350457b5..45e9a28c763a9f 100644 --- a/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py +++ b/metadata-ingestion/src/datahub/ingestion/sink/datahub_rest.py @@ -1,4 +1,5 @@ import concurrent.futures +import contextlib import functools import logging from dataclasses import dataclass @@ -111,13 +112,10 @@ def _write_done_callback( else: # trim exception stacktraces when reporting warnings if "stackTrace" in e.info: - try: + with contextlib.suppress(Exception): e.info["stackTrace"] = "\n".join( - e.info["stackTrace"].split("\n")[0:2] + e.info["stackTrace"].split("\n")[:2] ) - except Exception: - # ignore failures in trimming - pass record = record_envelope.record if isinstance(record, MetadataChangeProposalWrapper): # include information about the entity that failed @@ -159,3 +157,9 @@ def get_report(self) -> SinkReport: def close(self): self.executor.shutdown(wait=True) + + def __repr__(self) -> str: + return self.emitter.__repr__() + + def configured(self) -> str: + return self.__repr__() diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index a7171cfe42caf0..0667e840a6b336 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -716,7 +716,7 @@ def _create_profile_mcp( dataset_profile.rowCount = int( float(table_stats[self.source_config.profiling.row_count]) ) - if self.source_config.profiling.row_count in table_stats: + if self.source_config.profiling.column_count in table_stats: dataset_profile.columnCount = int( float(table_stats[self.source_config.profiling.column_count]) ) @@ -1045,6 +1045,10 @@ def get_dataset_properties() -> DatasetPropertiesClass: ) def get_s3_tags() -> Optional[GlobalTagsClass]: + # when TableType=VIRTUAL_VIEW the Location can be empty and we should + # return no tags rather than fail the entire ingestion + if table.get("StorageDescriptor", {}).get("Location") is None: + return None bucket_name = s3_util.get_bucket_name( table["StorageDescriptor"]["Location"] ) @@ -1059,7 +1063,7 @@ def get_s3_tags() -> Optional[GlobalTagsClass]: ] ) except self.s3_client.exceptions.ClientError: - logger.warn(f"No tags found for bucket={bucket_name}") + logger.warning(f"No tags found for bucket={bucket_name}") if self.source_config.use_s3_object_tags: key_prefix = s3_util.get_key_prefix( table["StorageDescriptor"]["Location"] @@ -1078,7 +1082,7 @@ def get_s3_tags() -> Optional[GlobalTagsClass]: else: # Unlike bucket tags, if an object does not have tags, it will just return an empty array # as opposed to an exception. - logger.warn( + logger.warning( f"No tags found for bucket={bucket_name} key={key_prefix}" ) if len(tags_to_add) == 0: @@ -1097,7 +1101,7 @@ def get_s3_tags() -> Optional[GlobalTagsClass]: [current_tag.tag for current_tag in current_tags.tags] ) else: - logger.warn( + logger.warning( "Could not connect to DatahubApi. No current tags to maintain" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/path_spec.py b/metadata-ingestion/src/datahub/ingestion/source/aws/path_spec.py new file mode 100644 index 00000000000000..0f3be16b396ff1 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/path_spec.py @@ -0,0 +1,188 @@ +import logging +import os +import re +from typing import Any, Dict, List, Optional, Tuple, Union + +import parse +import pydantic +from pydantic.fields import Field +from wcmatch import pathlib + +from datahub.configuration.common import ConfigModel +from datahub.ingestion.source.aws.s3_util import is_s3_uri + +# hide annoying debug errors from py4j +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + +SUPPORTED_FILE_TYPES: List[str] = ["csv", "tsv", "json", "parquet", "avro"] +SUPPORTED_COMPRESSIONS: List[str] = ["gz", "bz2"] + + +class PathSpec(ConfigModel): + class Config: + arbitrary_types_allowed = True + + include: str = Field( + description="Path to table (s3 or local file system). Name variable {table} is used to mark the folder with dataset. In absence of {table}, file level dataset will be created. Check below examples for more details." + ) + exclude: Optional[List[str]] = Field( + default=None, + description="list of paths in glob pattern which will be excluded while scanning for the datasets", + ) + file_types: List[str] = Field( + default=SUPPORTED_FILE_TYPES, + description="Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.", + ) + + default_extension: Optional[str] = Field( + description="For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.", + ) + + table_name: Optional[str] = Field( + default=None, + description="Display name of the dataset.Combination of named variableds from include path and strings", + ) + + enable_compression: bool = Field( + default=True, + description="Enable or disable processing compressed files. Currenly .gz and .bz files are supported.", + ) + + sample_files: bool = Field( + default=True, + description="Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled", + ) + + # to be set internally + _parsable_include: str + _compiled_include: parse.Parser + _glob_include: str + _is_s3: bool + + def allowed(self, path: str) -> bool: + logger.debug(f"Checking file to inclusion: {path}") + if not pathlib.PurePath(path).globmatch( + self._glob_include, flags=pathlib.GLOBSTAR + ): + return False + logger.debug(f"{path} matched include ") + if self.exclude: + for exclude_path in self.exclude: + if pathlib.PurePath(path).globmatch( + exclude_path, flags=pathlib.GLOBSTAR + ): + return False + logger.debug(f"{path} is not excluded") + ext = os.path.splitext(path)[1].strip(".") + + if (ext == "" and self.default_extension is None) and ( + ext != "*" and ext not in self.file_types + ): + return False + + logger.debug(f"{path} had selected extension {ext}") + logger.debug(f"{path} allowed for dataset creation") + return True + + def is_s3(self): + return self._is_s3 + + @classmethod + def get_parsable_include(cls, include: str) -> str: + parsable_include = include + for i in range(parsable_include.count("*")): + parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1) + return parsable_include + + def get_named_vars(self, path: str) -> Union[None, parse.Result, parse.Match]: + return self._compiled_include.parse(path) + + @pydantic.root_validator() + def validate_path_spec(cls, values: Dict) -> Dict[str, Any]: + + if "**" in values["include"]: + raise ValueError("path_spec.include cannot contain '**'") + + if values.get("file_types") is None: + values["file_types"] = SUPPORTED_FILE_TYPES + else: + for file_type in values["file_types"]: + if file_type not in SUPPORTED_FILE_TYPES: + raise ValueError( + f"file type {file_type} not in supported file types. Please specify one from {SUPPORTED_FILE_TYPES}" + ) + + if values.get("default_extension") is not None: + if values.get("default_extension") not in SUPPORTED_FILE_TYPES: + raise ValueError( + f"default extension {values.get('default_extension')} not in supported default file extension. Please specify one from {SUPPORTED_FILE_TYPES}" + ) + + include_ext = os.path.splitext(values["include"])[1].strip(".") + if ( + include_ext not in values["file_types"] + and include_ext != "*" + and not values["default_extension"] + and include_ext not in SUPPORTED_COMPRESSIONS + ): + raise ValueError( + f"file type specified ({include_ext}) in path_spec.include is not in specified file " + f'types. Please select one from {values.get("file_types")} or specify ".*" to allow all types' + ) + + values["_parsable_include"] = PathSpec.get_parsable_include(values["include"]) + logger.debug(f'Setting _parsable_include: {values.get("_parsable_include")}') + compiled_include_tmp = parse.compile(values["_parsable_include"]) + values["_compiled_include"] = compiled_include_tmp + logger.debug(f'Setting _compiled_include: {values["_compiled_include"]}') + values["_glob_include"] = re.sub(r"\{[^}]+\}", "*", values["include"]) + logger.debug(f'Setting _glob_include: {values.get("_glob_include")}') + + if values.get("table_name") is None: + if "{table}" in values["include"]: + values["table_name"] = "{table}" + else: + logger.debug(f"include fields: {compiled_include_tmp.named_fields}") + logger.debug( + f"table_name fields: {parse.compile(values['table_name']).named_fields}" + ) + if not all( + x in values["_compiled_include"].named_fields + for x in parse.compile(values["table_name"]).named_fields + ): + raise ValueError( + "Not all named variables used in path_spec.table_name are specified in " + "path_spec.include" + ) + + if values.get("exclude") is not None: + for exclude_path in values["exclude"]: + if len(parse.compile(exclude_path).named_fields) != 0: + raise ValueError( + "path_spec.exclude should not contain any named variables" + ) + + values["_is_s3"] = is_s3_uri(values["include"]) + if not values["_is_s3"]: + # Sampling only makes sense on s3 currently + values["sample_files"] = False + logger.debug(f'Setting _is_s3: {values.get("_is_s3")}') + return values + + def _extract_table_name(self, named_vars: dict) -> str: + if self.table_name is None: + raise ValueError("path_spec.table_name is not set") + return self.table_name.format_map(named_vars) + + def extract_table_name_and_path(self, path: str) -> Tuple[str, str]: + parsed_vars = self.get_named_vars(path) + if parsed_vars is None or "table" not in parsed_vars.named: + return os.path.basename(path), path + else: + include = self.include + depth = include.count("/", 0, include.find("{table}")) + table_path = ( + "/".join(path.split("/")[:depth]) + "/" + parsed_vars.named["table"] + ) + return self._extract_table_name(parsed_vars.named), table_path diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_boto_utils.py b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_boto_utils.py new file mode 100644 index 00000000000000..caec19d0fb2492 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_boto_utils.py @@ -0,0 +1,101 @@ +import logging +from typing import Iterable, Optional + +from datahub.emitter.mce_builder import make_tag_urn +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.aws.aws_common import AwsSourceConfig +from datahub.ingestion.source.aws.s3_util import ( + get_bucket_name, + get_bucket_relative_path, + is_s3_uri, +) +from datahub.metadata.schema_classes import GlobalTagsClass, TagAssociationClass + +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + + +def get_s3_tags( + bucket_name: str, + key_name: Optional[str], + dataset_urn: str, + aws_config: Optional[AwsSourceConfig], + ctx: PipelineContext, + use_s3_bucket_tags: Optional[bool] = False, + use_s3_object_tags: Optional[bool] = False, +) -> Optional[GlobalTagsClass]: + if aws_config is None: + raise ValueError("aws_config not set. Cannot browse s3") + new_tags = GlobalTagsClass(tags=[]) + tags_to_add = [] + if use_s3_bucket_tags: + s3 = aws_config.get_s3_resource() + bucket = s3.Bucket(bucket_name) + try: + tags_to_add.extend( + [ + make_tag_urn(f"""{tag["Key"]}:{tag["Value"]}""") + for tag in bucket.Tagging().tag_set + ] + ) + except s3.meta.client.exceptions.ClientError: + logger.warn(f"No tags found for bucket={bucket_name}") + + if use_s3_object_tags and key_name is not None: + s3_client = aws_config.get_s3_client() + object_tagging = s3_client.get_object_tagging(Bucket=bucket_name, Key=key_name) + tag_set = object_tagging["TagSet"] + if tag_set: + tags_to_add.extend( + [make_tag_urn(f"""{tag["Key"]}:{tag["Value"]}""") for tag in tag_set] + ) + else: + # Unlike bucket tags, if an object does not have tags, it will just return an empty array + # as opposed to an exception. + logger.warn(f"No tags found for bucket={bucket_name} key={key_name}") + if len(tags_to_add) == 0: + return None + if ctx.graph is not None: + logger.debug("Connected to DatahubApi, grabbing current tags to maintain.") + current_tags: Optional[GlobalTagsClass] = ctx.graph.get_aspect_v2( + entity_urn=dataset_urn, + aspect="globalTags", + aspect_type=GlobalTagsClass, + ) + if current_tags: + tags_to_add.extend([current_tag.tag for current_tag in current_tags.tags]) + else: + logger.warn("Could not connect to DatahubApi. No current tags to maintain") + # Remove duplicate tags + tags_to_add = list(set(tags_to_add)) + new_tags = GlobalTagsClass( + tags=[TagAssociationClass(tag_to_add) for tag_to_add in tags_to_add] + ) + return new_tags + + +def list_folders_path( + s3_uri: str, aws_config: Optional[AwsSourceConfig] +) -> Iterable[str]: + if not is_s3_uri(s3_uri): + raise ValueError("Not a s3 URI: " + s3_uri) + if aws_config is None: + raise ValueError("aws_config not set. Cannot browse s3") + bucket_name = get_bucket_name(s3_uri) + prefix = get_bucket_relative_path(s3_uri) + yield from list_folders(bucket_name, prefix, aws_config) + + +def list_folders( + bucket_name: str, prefix: str, aws_config: Optional[AwsSourceConfig] +) -> Iterable[str]: + if aws_config is None: + raise ValueError("aws_config not set. Cannot browse s3") + s3_client = aws_config.get_s3_client() + paginator = s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter="/"): + for o in page.get("CommonPrefixes", []): + folder: str = str(o.get("Prefix")) + if folder.endswith("/"): + folder = folder[:-1] + yield f"{folder}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py index a8e4587a917831..a09b21a4c8da7e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py @@ -1,7 +1,11 @@ +import logging import os S3_PREFIXES = ["s3://", "s3n://", "s3a://"] +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + def is_s3_uri(uri: str) -> bool: return any(uri.startswith(prefix) for prefix in S3_PREFIXES) @@ -24,7 +28,6 @@ def get_bucket_relative_path(s3_uri: str) -> str: def make_s3_urn(s3_uri: str, env: str) -> str: - s3_name = strip_s3_prefix(s3_uri) if s3_name.endswith("/"): diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py new file mode 100644 index 00000000000000..1ac80968739f16 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -0,0 +1,687 @@ +import csv +import time +from dataclasses import dataclass +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union + +from datahub.configuration.common import ConfigurationError +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SupportStatus, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source_config.csv_enricher import CSVEnricherConfig +from datahub.metadata.schema_classes import ( + AuditStampClass, + ChangeTypeClass, + DomainsClass, + EditableDatasetPropertiesClass, + EditableSchemaFieldInfoClass, + EditableSchemaMetadataClass, + GlobalTagsClass, + GlossaryTermAssociationClass, + GlossaryTermsClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + TagAssociationClass, +) +from datahub.utilities.urns.dataset_urn import DatasetUrn +from datahub.utilities.urns.urn import Urn + +SCHEMA_ASPECT_NAME = "editableSchemaMetadata" +DATASET_ENTITY_TYPE = "dataset" +GLOSSARY_TERMS_ASPECT_NAME = "glossaryTerms" +TAGS_ASPECT_NAME = "globalTags" +OWNERSHIP_ASPECT_NAME = "ownership" +EDITABLE_DATASET_PROPERTIES_ASPECT_NAME = "editableDatasetProperties" +ACTOR = "urn:li:corpuser:ingestion" +DOMAIN_ASPECT_NAME = "domains" + + +def get_audit_stamp() -> AuditStampClass: + now = int(time.time() * 1000) + return AuditStampClass(now, ACTOR) + + +def maybe_remove_prefix(s: str, prefix: str) -> str: + if not s.startswith(prefix): + return s + return s[len(prefix) :] + + +def maybe_remove_suffix(s: str, suffix: str) -> str: + if not s.endswith(suffix): + return s + return s[: -len(suffix)] + + +def sanitize_array_string(s: str) -> str: + return maybe_remove_suffix(maybe_remove_prefix(s, "["), "]") + + +@dataclass +class SubResourceRow: + entity_urn: str + field_path: str + term_associations: List[GlossaryTermAssociationClass] + tag_associations: List[TagAssociationClass] + description: Optional[str] + domain: Optional[str] + + +@dataclass +class CSVEnricherReport(SourceReport): + num_glossary_term_workunits_produced: int = 0 + num_tag_workunits_produced: int = 0 + num_owners_workunits_produced: int = 0 + num_description_workunits_produced: int = 0 + num_editable_schema_metadata_workunits_produced: int = 0 + num_domain_workunits_produced: int = 0 + + +@platform_name("CSV") +@config_class(CSVEnricherConfig) +@support_status(SupportStatus.INCUBATING) +class CSVEnricherSource(Source): + """ + This plugin is used to apply glossary terms, tags, owners and domain at the entity level. It can also be used to apply tags + and glossary terms at the column level. These values are read from a CSV file and can be used to either overwrite + or append the above aspects to entities. + + The format of the CSV must be like so, with a few example rows. + + |resource |subresource|glossary_terms |tags |owners |ownership_type |description |domain | + |----------------------------------------------------------------|-----------|------------------------------------|-------------------|---------------------------------------------------|---------------|---------------|---------------------------| + |urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)| |[urn:li:glossaryTerm:AccountBalance]|[urn:li:tag:Legacy]|[urn:li:corpuser:datahub|urn:li:corpuser:jdoe]|TECHNICAL_OWNER|new description|urn:li:domain:Engineering | + |urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)|field_foo |[urn:li:glossaryTerm:AccountBalance]| | | |field_foo! | | + |urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)|field_bar | |[urn:li:tag:Legacy]| | |field_bar? | | + + Note that the first row does not have a subresource populated. That means any glossary terms, tags, and owners will + be applied at the entity field. If a subresource IS populated (as it is for the second and third rows), glossary + terms and tags will be applied on the subresource. Every row MUST have a resource. Also note that owners can only + be applied at the resource level and will be ignored if populated for a row with a subresource. + """ + + def __init__(self, config: CSVEnricherConfig, ctx: PipelineContext): + super().__init__(ctx) + self.config: CSVEnricherConfig = config + self.ctx: PipelineContext = ctx + self.report: CSVEnricherReport = CSVEnricherReport() + # Map from entity urn to a list of SubResourceRow. + self.editable_schema_metadata_map: Dict[str, List[SubResourceRow]] = {} + self.should_overwrite: bool = self.config.write_semantics == "OVERRIDE" + if not self.should_overwrite and not self.ctx.graph: + raise ConfigurationError( + "With PATCH semantics, the csv-enricher source requires a datahub_api to connect to. " + "Consider using the datahub-rest sink or provide a datahub_api: configuration on your ingestion recipe." + ) + + def get_resource_glossary_terms_work_unit( + self, + entity_urn: str, + entity_type: str, + term_associations: List[GlossaryTermAssociationClass], + ) -> Optional[MetadataWorkUnit]: + # Check if there are glossary terms to add. If not, return None. + if len(term_associations) <= 0: + return None + + current_terms: Optional[GlossaryTermsClass] = None + if self.ctx.graph and not self.should_overwrite: + # Get the existing terms for the entity from the DataHub graph + current_terms = self.ctx.graph.get_glossary_terms(entity_urn=entity_urn) + + if not current_terms: + # If we want to overwrite or there are no existing terms, create a new GlossaryTerms object + current_terms = GlossaryTermsClass(term_associations, get_audit_stamp()) + else: + current_term_urns: Set[str] = set( + [term.urn for term in current_terms.terms] + ) + term_associations_filtered: List[GlossaryTermAssociationClass] = [ + association + for association in term_associations + if association.urn not in current_term_urns + ] + # If there are no new glossary terms to add, we don't need to emit a work unit. + if len(term_associations_filtered) <= 0: + return None + + # Add any terms that don't already exist in the existing GlossaryTerms object to the object + current_terms.terms.extend(term_associations_filtered) + + terms_mcpw: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType=entity_type, + entityUrn=entity_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName=GLOSSARY_TERMS_ASPECT_NAME, + aspect=current_terms, + ) + terms_wu: MetadataWorkUnit = MetadataWorkUnit( + id=f"{entity_urn}-{GLOSSARY_TERMS_ASPECT_NAME}", + mcp=terms_mcpw, + ) + return terms_wu + + def get_resource_tags_work_unit( + self, + entity_urn: str, + entity_type: str, + tag_associations: List[TagAssociationClass], + ) -> Optional[MetadataWorkUnit]: + # Check if there are tags to add. If not, return None. + if len(tag_associations) <= 0: + return None + + current_tags: Optional[GlobalTagsClass] = None + if self.ctx.graph and not self.should_overwrite: + # Get the existing tags for the entity from the DataHub graph + current_tags = self.ctx.graph.get_tags(entity_urn=entity_urn) + + if not current_tags: + # If we want to overwrite or there are no existing tags, create a new GlobalTags object + current_tags = GlobalTagsClass(tag_associations) + else: + current_tag_urns: Set[str] = set([tag.tag for tag in current_tags.tags]) + tag_associations_filtered: List[TagAssociationClass] = [ + association + for association in tag_associations + if association.tag not in current_tag_urns + ] + # If there are no new tags to add, we don't need to emit a work unit. + if len(tag_associations_filtered) <= 0: + return None + + # Add any terms that don't already exist in the existing GlobalTags object to the object + current_tags.tags.extend(tag_associations_filtered) + + tags_mcpw: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType=entity_type, + entityUrn=entity_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName=TAGS_ASPECT_NAME, + aspect=current_tags, + ) + tags_wu: MetadataWorkUnit = MetadataWorkUnit( + id=f"{entity_urn}-{TAGS_ASPECT_NAME}", + mcp=tags_mcpw, + ) + return tags_wu + + def get_resource_owners_work_unit( + self, + entity_urn: str, + entity_type: str, + owners: List[OwnerClass], + ) -> Optional[MetadataWorkUnit]: + # Check if there are owners to add. If not, return None. + if len(owners) <= 0: + return None + + current_ownership: Optional[OwnershipClass] = None + if self.ctx.graph and not self.should_overwrite: + # Get the existing owner for the entity from the DataHub graph + current_ownership = self.ctx.graph.get_ownership(entity_urn=entity_urn) + + if not current_ownership: + # If we want to overwrite or there are no existing tags, create a new GlobalTags object + current_ownership = OwnershipClass(owners, get_audit_stamp()) + else: + current_owner_urns: Set[str] = set( + [owner.owner for owner in current_ownership.owners] + ) + owners_filtered: List[OwnerClass] = [ + owner for owner in owners if owner.owner not in current_owner_urns + ] + # If there are no new owners to add, we don't need to emit a work unit. + if len(owners_filtered) <= 0: + return None + + # Add any terms that don't already exist in the existing GlobalTags object to the object + current_ownership.owners.extend(owners_filtered) + + owners_mcpw: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType=entity_type, + entityUrn=entity_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName=OWNERSHIP_ASPECT_NAME, + aspect=current_ownership, + ) + owners_wu: MetadataWorkUnit = MetadataWorkUnit( + id=f"{entity_urn}-{OWNERSHIP_ASPECT_NAME}", + mcp=owners_mcpw, + ) + return owners_wu + + def get_resource_domain_work_unit( + self, + entity_urn: str, + entity_type: str, + domain: Optional[str], + ) -> Optional[MetadataWorkUnit]: + # Check if there is a domain to add. If not, return None. + if not domain: + return None + + current_domain: Optional[DomainsClass] = None + if self.ctx.graph and not self.should_overwrite: + # Get the existing domain for the entity from the DataHub graph + current_domain = self.ctx.graph.get_domain(entity_urn=entity_urn) + + if not current_domain: + # If we want to overwrite or there is no existing domain, create a new object + current_domain = DomainsClass([domain]) + + domain_mcpw: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType=entity_type, + entityUrn=entity_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName=DOMAIN_ASPECT_NAME, + aspect=current_domain, + ) + domain_wu: MetadataWorkUnit = MetadataWorkUnit( + id=f"{entity_urn}-{DOMAIN_ASPECT_NAME}", + mcp=domain_mcpw, + ) + return domain_wu + + def get_resource_description_work_unit( + self, + entity_urn: str, + entity_type: str, + description: Optional[str], + ) -> Optional[MetadataWorkUnit]: + # Check if there is a description to add. If not, return None. + if not description: + return None + + # If the description is empty, return None. + if len(description) <= 0: + return None + + current_editable_properties: Optional[EditableDatasetPropertiesClass] = None + if self.ctx.graph and not self.should_overwrite: + # Get the existing editable properties for the entity from the DataHub graph + current_editable_properties = self.ctx.graph.get_aspect_v2( + entity_urn=entity_urn, + aspect=EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, + aspect_type=EditableDatasetPropertiesClass, + ) + + if not current_editable_properties: + # If we want to overwrite or there are no existing editable dataset properties, create a new object + current_editable_properties = EditableDatasetPropertiesClass( + created=get_audit_stamp(), + lastModified=get_audit_stamp(), + description=description, + ) + else: + current_editable_properties.description = description + current_editable_properties.lastModified = get_audit_stamp() + + description_mcpw: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityType=entity_type, + entityUrn=entity_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName=EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, + aspect=current_editable_properties, + ) + description_wu: MetadataWorkUnit = MetadataWorkUnit( + id=f"{entity_urn}-{EDITABLE_DATASET_PROPERTIES_ASPECT_NAME}", + mcp=description_mcpw, + ) + return description_wu + + def get_resource_workunits( + self, + entity_urn: str, + entity_type: str, + term_associations: List[GlossaryTermAssociationClass], + tag_associations: List[TagAssociationClass], + owners: List[OwnerClass], + domain: Optional[str], + description: Optional[str], + ) -> Iterable[MetadataWorkUnit]: + maybe_terms_wu: Optional[ + MetadataWorkUnit + ] = self.get_resource_glossary_terms_work_unit( + entity_urn=entity_urn, + entity_type=entity_type, + term_associations=term_associations, + ) + if maybe_terms_wu: + self.report.num_glossary_term_workunits_produced += 1 + self.report.report_workunit(maybe_terms_wu) + yield maybe_terms_wu + + maybe_tags_wu: Optional[MetadataWorkUnit] = self.get_resource_tags_work_unit( + entity_urn=entity_urn, + entity_type=entity_type, + tag_associations=tag_associations, + ) + if maybe_tags_wu: + self.report.num_tag_workunits_produced += 1 + self.report.report_workunit(maybe_tags_wu) + yield maybe_tags_wu + + maybe_owners_wu: Optional[ + MetadataWorkUnit + ] = self.get_resource_owners_work_unit( + entity_urn=entity_urn, + entity_type=entity_type, + owners=owners, + ) + if maybe_owners_wu: + self.report.num_owners_workunits_produced += 1 + self.report.report_workunit(maybe_owners_wu) + yield maybe_owners_wu + + maybe_domain_wu: Optional[ + MetadataWorkUnit + ] = self.get_resource_domain_work_unit( + entity_urn=entity_urn, + entity_type=entity_type, + domain=domain, + ) + if maybe_domain_wu: + self.report.num_domain_workunits_produced += 1 + self.report.report_workunit(maybe_domain_wu) + yield maybe_domain_wu + + maybe_description_wu: Optional[ + MetadataWorkUnit + ] = self.get_resource_description_work_unit( + entity_urn=entity_urn, + entity_type=entity_type, + description=description, + ) + if maybe_description_wu: + self.report.num_description_workunits_produced += 1 + self.report.report_workunit(maybe_description_wu) + yield maybe_description_wu + + def process_sub_resource_row( + self, + sub_resource_row: SubResourceRow, + current_editable_schema_metadata: EditableSchemaMetadataClass, + needs_write: bool, + ) -> Tuple[EditableSchemaMetadataClass, bool]: + field_path: str = sub_resource_row.field_path + term_associations: List[ + GlossaryTermAssociationClass + ] = sub_resource_row.term_associations + tag_associations: List[TagAssociationClass] = sub_resource_row.tag_associations + description: Optional[str] = sub_resource_row.description + has_terms: bool = len(term_associations) > 0 + has_tags: bool = len(tag_associations) > 0 + has_description: bool = description is not None and len(description) > 0 + + # We can skip this row if there are no tags, terms or description to edit. + if not has_tags and not has_terms and not has_description: + return current_editable_schema_metadata, needs_write + + # Objects that may or not be written depending on which conditions get triggered. + field_info_to_set = EditableSchemaFieldInfoClass(fieldPath=field_path) + terms_aspect = ( + GlossaryTermsClass(term_associations, get_audit_stamp()) + if has_terms + else None + ) + if terms_aspect: + field_info_to_set.glossaryTerms = terms_aspect + tags_aspect = GlobalTagsClass(tag_associations) if has_tags else None + if tags_aspect: + field_info_to_set.globalTags = tags_aspect + if has_description: + field_info_to_set.description = description + + # Boolean field to tell whether we have found a field match. + field_match = False + for field_info in current_editable_schema_metadata.editableSchemaFieldInfo: + if ( + DatasetUrn._get_simple_field_path_from_v2_field_path( + field_info.fieldPath + ) + == field_path + ): + # we have some editable schema metadata for this field + field_match = True + if has_terms: + if field_info.glossaryTerms and not self.should_overwrite: + current_term_urns = set( + [term.urn for term in field_info.glossaryTerms.terms] + ) + term_associations_filtered = [ + association + for association in term_associations + if association.urn not in current_term_urns + ] + if len(term_associations_filtered) > 0: + field_info.glossaryTerms.terms.extend( + term_associations_filtered + ) + needs_write = True + else: + field_info.glossaryTerms = terms_aspect + needs_write = True + + if has_tags: + if field_info.globalTags and not self.should_overwrite: + current_tag_urns = set( + [tag.tag for tag in field_info.globalTags.tags] + ) + tag_associations_filtered = [ + association + for association in tag_associations + if association.tag not in current_tag_urns + ] + if len(tag_associations_filtered) > 0: + field_info.globalTags.tags.extend(tag_associations_filtered) + needs_write = True + else: + field_info.globalTags = tags_aspect + needs_write = True + + if has_description: + field_info.description = description + needs_write = True + + if not field_match: + # this field isn't present in the editable schema metadata aspect, add it + current_editable_schema_metadata.editableSchemaFieldInfo.append( + field_info_to_set + ) + needs_write = True + return current_editable_schema_metadata, needs_write + + def get_sub_resource_work_units(self) -> Iterable[MetadataWorkUnit]: + # Iterate over the map + for entity_urn in self.editable_schema_metadata_map: + # Boolean field to tell whether we need to write an MCPW. + needs_write = False + + current_editable_schema_metadata: Optional[ + EditableSchemaMetadataClass + ] = None + if self.ctx.graph and not self.should_overwrite: + # Fetch the current editable schema metadata + current_editable_schema_metadata = self.ctx.graph.get_aspect_v2( + entity_urn=entity_urn, + aspect=SCHEMA_ASPECT_NAME, + aspect_type=EditableSchemaMetadataClass, + ) + + # Create a new editable schema metadata for the dataset if it doesn't exist + if not current_editable_schema_metadata: + current_editable_schema_metadata = EditableSchemaMetadataClass( + editableSchemaFieldInfo=[], + created=get_audit_stamp(), + ) + needs_write = True + + # Iterate over each sub resource row + for sub_resource_row in self.editable_schema_metadata_map[ + entity_urn + ]: # type: SubResourceRow + ( + current_editable_schema_metadata, + needs_write, + ) = self.process_sub_resource_row( + sub_resource_row, current_editable_schema_metadata, needs_write + ) + + # Write an MCPW if needed. + if needs_write: + editable_schema_metadata_mcpw: MetadataChangeProposalWrapper = ( + MetadataChangeProposalWrapper( + entityType=DATASET_ENTITY_TYPE, + changeType=ChangeTypeClass.UPSERT, + entityUrn=entity_urn, + aspectName=SCHEMA_ASPECT_NAME, + aspect=current_editable_schema_metadata, + ) + ) + wu: MetadataWorkUnit = MetadataWorkUnit( + id=f"{entity_urn}-{SCHEMA_ASPECT_NAME}", + mcp=editable_schema_metadata_mcpw, + ) + yield wu + + def maybe_extract_glossary_terms( + self, row: Dict[str, str] + ) -> List[GlossaryTermAssociationClass]: + if not row["glossary_terms"]: + return [] + + # Sanitizing the terms string to just get the list of term urns + terms_array_string = sanitize_array_string(row["glossary_terms"]) + term_urns: List[str] = terms_array_string.split(self.config.array_delimiter) + + term_associations: List[GlossaryTermAssociationClass] = [ + GlossaryTermAssociationClass(term) for term in term_urns + ] + return term_associations + + def maybe_extract_tags(self, row: Dict[str, str]) -> List[TagAssociationClass]: + if not row["tags"]: + return [] + + # Sanitizing the tags string to just get the list of tag urns + tags_array_string = sanitize_array_string(row["tags"]) + tag_urns: List[str] = tags_array_string.split(self.config.array_delimiter) + + tag_associations: List[TagAssociationClass] = [ + TagAssociationClass(tag) for tag in tag_urns + ] + return tag_associations + + def maybe_extract_owners( + self, row: Dict[str, str], is_resource_row: bool + ) -> List[OwnerClass]: + if not is_resource_row: + return [] + + if not row["owners"]: + return [] + + # Getting the ownership type + ownership_type: Union[str, OwnershipTypeClass] = ( + row["ownership_type"] if row["ownership_type"] else OwnershipTypeClass.NONE + ) + + # Sanitizing the owners string to just get the list of owner urns + owners_array_string: str = sanitize_array_string(row["owners"]) + owner_urns: List[str] = owners_array_string.split(self.config.array_delimiter) + + owners: List[OwnerClass] = [ + OwnerClass(owner_urn, type=ownership_type) for owner_urn in owner_urns + ] + return owners + + def get_workunits(self) -> Iterable[MetadataWorkUnit]: + with open(self.config.filename, "r") as f: + rows = csv.DictReader(f, delimiter=self.config.delimiter) + for row in rows: + # We need the resource to move forward + if not row["resource"]: + continue + + is_resource_row: bool = not row["subresource"] + + entity_urn = row["resource"] + entity_type = Urn.create_from_string(row["resource"]).get_type() + + term_associations: List[ + GlossaryTermAssociationClass + ] = self.maybe_extract_glossary_terms(row) + + tag_associations: List[TagAssociationClass] = self.maybe_extract_tags( + row + ) + + owners: List[OwnerClass] = self.maybe_extract_owners( + row, is_resource_row + ) + + domain: Optional[str] = ( + row["domain"] + if row["domain"] and entity_type == DATASET_ENTITY_TYPE + else None + ) + + description: Optional[str] = ( + row["description"] + if row["description"] and entity_type == DATASET_ENTITY_TYPE + else None + ) + + if is_resource_row: + for wu in self.get_resource_workunits( + entity_urn=entity_urn, + entity_type=entity_type, + term_associations=term_associations, + tag_associations=tag_associations, + owners=owners, + domain=domain, + description=description, + ): + yield wu + + # If this row is not applying changes at the resource level, modify the EditableSchemaMetadata map. + else: + # Only dataset sub-resources are currently supported. + if entity_type != DATASET_ENTITY_TYPE: + continue + + field_path = row["subresource"] + if entity_urn not in self.editable_schema_metadata_map: + self.editable_schema_metadata_map[entity_urn] = [] + # Add the row to the map from entity (dataset) to SubResource rows. We cannot emit work units for + # EditableSchemaMetadata until we parse the whole CSV due to read-modify-write issues. + self.editable_schema_metadata_map[entity_urn].append( + SubResourceRow( + entity_urn=entity_urn, + field_path=field_path, + term_associations=term_associations, + tag_associations=tag_associations, + description=description, + domain=domain, + ) + ) + + # Yield sub resource work units once the map has been fully populated. + for wu in self.get_sub_resource_work_units(): + self.report.num_editable_schema_metadata_workunits_produced += 1 + self.report.report_workunit(wu) + yield wu + + def get_report(self): + return self.report + + def close(self): + pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake/data_lake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake/data_lake_utils.py new file mode 100644 index 00000000000000..02e516c8a2d052 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake/data_lake_utils.py @@ -0,0 +1,118 @@ +import logging +from typing import Iterable, List, Optional + +from datahub.emitter.mcp_builder import ( + FolderKey, + KeyType, + PlatformKey, + S3BucketKey, + add_dataset_to_container, + gen_containers, +) +from datahub.ingestion.api.workunit import MetadataWorkUnit + +# hide annoying debug errors from py4j +from datahub.ingestion.source.aws.s3_util import ( + get_bucket_name, + get_bucket_relative_path, + is_s3_uri, +) + +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + + +class ContainerWUCreator: + processed_containers: List[str] + + def __init__(self, platform, platform_instance, env): + self.processed_containers = [] + self.platform = platform + self.instance = env if platform_instance is None else platform_instance + + def create_emit_containers( + self, + container_key: KeyType, + name: str, + sub_types: List[str], + parent_container_key: Optional[PlatformKey] = None, + domain_urn: Optional[str] = None, + ) -> Iterable[MetadataWorkUnit]: + if container_key.guid() not in self.processed_containers: + container_wus = gen_containers( + container_key=container_key, + name=name, + sub_types=sub_types, + parent_container_key=parent_container_key, + domain_urn=domain_urn, + ) + self.processed_containers.append(container_key.guid()) + logger.debug(f"Creating container with key: {container_key}") + for wu in container_wus: + yield wu + + def gen_folder_key(self, abs_path): + return FolderKey( + platform=self.platform, + instance=self.instance, + folder_abs_path=abs_path, + ) + + def gen_bucket_key(self, name): + return S3BucketKey( + platform="s3", + instance=self.instance, + bucket_name=name, + ) + + def create_container_hierarchy( + self, path: str, is_s3: bool, dataset_urn: str + ) -> Iterable[MetadataWorkUnit]: + logger.debug(f"Creating containers for {dataset_urn}") + base_full_path = path + parent_key = None + if is_s3_uri(path): + bucket_name = get_bucket_name(path) + bucket_key = self.gen_bucket_key(bucket_name) + yield from self.create_emit_containers( + container_key=bucket_key, + name=bucket_name, + sub_types=["S3 bucket"], + parent_container_key=None, + ) + parent_key = bucket_key + base_full_path = get_bucket_relative_path(path) + + parent_folder_path = ( + base_full_path[: base_full_path.rfind("/")] + if base_full_path.rfind("/") != -1 + else "" + ) + + # Dataset is in the root folder + if not parent_folder_path and parent_key is None: + logger.warning( + f"Failed to associate Dataset ({dataset_urn}) with container" + ) + return + + for folder in parent_folder_path.split("/"): + abs_path = folder + if parent_key: + prefix: str = "" + if isinstance(parent_key, S3BucketKey): + prefix = parent_key.bucket_name + elif isinstance(parent_key, FolderKey): + prefix = parent_key.folder_abs_path + abs_path = prefix + "/" + folder + folder_key = self.gen_folder_key(abs_path) + yield from self.create_emit_containers( + container_key=folder_key, + name=folder, + sub_types=["Folder"], + parent_container_key=parent_key, + ) + parent_key = folder_key + + assert parent_key is not None + yield from add_dataset_to_container(parent_key, dataset_urn) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index d257615133ab3d..1cd7b881d49c1f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -2,12 +2,26 @@ import logging import re from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, cast +from datetime import datetime +from enum import Enum +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + Union, + cast, +) from urllib.parse import urlparse import dateutil.parser import requests -from pydantic import validator +from cached_property import cached_property +from pydantic import BaseModel, root_validator, validator from pydantic.fields import Field from datahub.configuration.common import AllowDenyPattern, ConfigurationError @@ -30,9 +44,12 @@ POSTGRES_TYPES_MAP, SNOWFLAKE_TYPES_MAP, SPARK_SQL_TYPES_MAP, + TRINO_SQL_TYPES_MAP, resolve_postgres_modified_type, + resolve_trino_modified_type, ) from datahub.ingestion.source.state.checkpoint import Checkpoint +from datahub.ingestion.source.state.dbt_state import DbtCheckpointState from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, ) @@ -67,7 +84,21 @@ TimeTypeClass, ) from datahub.metadata.schema_classes import ( + AssertionInfoClass, + AssertionResultClass, + AssertionResultTypeClass, + AssertionRunEventClass, + AssertionRunStatusClass, + AssertionStdAggregationClass, + AssertionStdOperatorClass, + AssertionStdParameterClass, + AssertionStdParametersClass, + AssertionStdParameterTypeClass, + AssertionTypeClass, ChangeTypeClass, + DataPlatformInstanceClass, + DatasetAssertionInfoClass, + DatasetAssertionScopeClass, DatasetPropertiesClass, GlobalTagsClass, GlossaryTermsClass, @@ -105,6 +136,89 @@ def report_stale_entity_soft_deleted(self, urn: str) -> None: self.soft_deleted_stale_entities.append(urn) +class EmitDirective(Enum): + """A holder for directives for emission for specific types of entities""" + + YES = "YES" # Okay to emit for this type + NO = "NO" # Do not emit for this type + ONLY = "ONLY" # Only emit metadata for this type and no others + + +class DBTEntitiesEnabled(BaseModel): + """Controls which dbt entities are going to be emitted by this source""" + + class Config: + arbitrary_types_allowed = True # needed to allow cached_property to work + keep_untouched = ( + cached_property, + ) # needed to allow cached_property to work. See https://github.com/samuelcolvin/pydantic/issues/1241 for more info. + + models: EmitDirective = Field( + "Yes", description="Emit metadata for dbt models when set to Yes or Only" + ) + sources: EmitDirective = Field( + "Yes", description="Emit metadata for dbt sources when set to Yes or Only" + ) + seeds: EmitDirective = Field( + "Yes", description="Emit metadata for dbt seeds when set to Yes or Only" + ) + test_definitions: EmitDirective = Field( + "Yes", + description="Emit metadata for test definitions when enabled when set to Yes or Only", + ) + test_results: EmitDirective = Field( + "Yes", description="Emit metadata for test results when set to Yes or Only" + ) + + @validator("*", pre=True, always=True) + def to_upper(cls, v): + return v.upper() if isinstance(v, str) else v + + @root_validator + def only_one_can_be_set_to_only(cls, values): + only_values = [k for k in values if values.get(k) == EmitDirective.ONLY] + if len(only_values) > 1: + raise ValueError( + f"Cannot have more than 1 type of entity emission set to ONLY. Found {only_values}" + ) + return values + + def _any_other_only_set(self, attribute: str) -> bool: + """Return true if any attribute other than the one passed in is set to ONLY""" + other_onlies = [ + k + for k, v in self.__dict__.items() + if k != attribute and v == EmitDirective.ONLY + ] + return len(other_onlies) != 0 + + @cached_property # type: ignore + def node_type_emit_decision_cache(self) -> Dict[str, bool]: + node_type_for_field_map = { + "models": "model", + "sources": "source", + "seeds": "seed", + "test_definitions": "test", + } + return { + node_type_for_field_map[k]: False + if self._any_other_only_set(k) + or self.__getattribute__(k) == EmitDirective.NO + else True + for k in ["models", "sources", "seeds", "test_definitions"] + } + + def can_emit_node_type(self, node_type: str) -> bool: + return self.node_type_emit_decision_cache.get(node_type, False) + + @property + def can_emit_test_results(self) -> bool: + return ( + not self._any_other_only_set("test_results") + and self.test_results != EmitDirective.NO + ) + + class DBTConfig(StatefulIngestionConfigBase): manifest_path: str = Field( description="Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json Note this can be a local file or a URI." @@ -116,12 +230,20 @@ class DBTConfig(StatefulIngestionConfigBase): default=None, description="Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not specified, last-modified fields will not be populated. Note this can be a local file or a URI.", ) + test_results_path: Optional[str] = Field( + default=None, + description="Path to output of dbt test run as run_results file in JSON format. See https://docs.getdbt.com/reference/artifacts/run-results-json. If not specified, test execution results will not be populated in DataHub.", + ) env: str = Field( default=mce_builder.DEFAULT_ENV, description="Environment to use in namespace when constructing URNs.", ) target_platform: str = Field( - description="The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.)" + description="The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.)", + ) + target_platform_instance: Optional[str] = Field( + default=None, + description="The platform instance for the platform that dbt is operating on. Use this if you have multiple instances of the same platform (e.g. redshift) and need to distinguish between them.", ) load_schemas: bool = Field( default=True, @@ -133,7 +255,11 @@ class DBTConfig(StatefulIngestionConfigBase): ) node_type_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), - description="regex patterns for dbt nodes to filter in ingestion.", + description="Deprecated: use entities_enabled instead. Regex patterns for dbt nodes to filter in ingestion.", + ) + entities_enabled: DBTEntitiesEnabled = Field( + DBTEntitiesEnabled(), + description="Controls for enabling / disabling metadata emission for different dbt entities (models, test definitions, test results, etc.)", ) tag_prefix: str = Field( default=f"{DBT_PLATFORM}:", description="Prefix added to tags during ingestion." @@ -178,6 +304,14 @@ class DBTConfig(StatefulIngestionConfigBase): default=None, description="When fetching manifest files from s3, configuration for aws connection details", ) + delete_tests_as_datasets: bool = Field( + False, + description="Prior to version 0.8.38, dbt tests were represented as datasets. If you ingested dbt tests before, set this flag to True (just needed once) to soft-delete tests that were generated as datasets by previous ingestion.", + ) + backcompat_skip_source_on_lineage_edge: bool = Field( + False, + description="Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.", + ) @property def s3_client(self): @@ -216,14 +350,42 @@ def aws_connection_needed_if_s3_uris_present( uri_containing_fields = [ f for f in ["manifest_path", "catalog_path", "sources_path"] - if values.get(f, "").startswith("s3://") + if (values.get(f) or "").startswith("s3://") ] + if uri_containing_fields and not aws_connection: raise ValueError( f"Please provide aws_connection configuration, since s3 uris have been provided in fields {uri_containing_fields}" ) return aws_connection + @validator("meta_mapping") + def meta_mapping_validator( + cls, meta_mapping: Dict[str, Any], values: Dict, **kwargs: Any + ) -> Dict[str, Any]: + for k, v in meta_mapping.items(): + if "match" not in v: + raise ValueError( + f"meta_mapping section {k} doesn't have a match clause." + ) + if "operation" not in v: + raise ValueError( + f"meta_mapping section {k} doesn't have an operation clause." + ) + if v["operation"] == "add_owner": + owner_category = v["config"].get("owner_category") + if owner_category: + allowed_categories = [ + value + for name, value in vars(OwnershipTypeClass).items() + if not name.startswith("_") + ] + if (owner_category.upper()) not in allowed_categories: + raise ValueError( + f"Owner category {owner_category} is not one of {allowed_categories}" + ) + return meta_mapping + @dataclass class DBTColumn: @@ -244,10 +406,12 @@ class DBTNode: database: Optional[str] schema: str name: str # name, identifier + alias: Optional[str] # alias if present comment: str description: str raw_sql: Optional[str] + dbt_adapter: str dbt_name: str dbt_file_path: str @@ -266,6 +430,7 @@ class DBTNode: query_tag: Dict[str, Any] = field(default_factory=dict) tags: List[str] = field(default_factory=list) + compiled_sql: Optional[str] = None def __repr__(self): fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items()) @@ -303,12 +468,14 @@ def extract_dbt_entities( all_manifest_entities: Dict[str, Dict[str, Any]], all_catalog_entities: Dict[str, Dict[str, Any]], sources_results: List[Dict[str, Any]], + manifest_adapter: str, load_schemas: bool, use_identifiers: bool, tag_prefix: str, node_type_pattern: AllowDenyPattern, report: DBTSourceReport, node_name_pattern: AllowDenyPattern, + entities_enabled: DBTEntitiesEnabled, ) -> List[DBTNode]: sources_by_id = {x["unique_id"]: x for x in sources_results} @@ -316,13 +483,21 @@ def extract_dbt_entities( for key, manifest_node in all_manifest_entities.items(): # check if node pattern allowed based on config file if not node_type_pattern.allowed(manifest_node["resource_type"]): + logger.debug( + f"Not extracting dbt entity {key} since node type {manifest_node['resource_type']} is disabled" + ) continue name = manifest_node["name"] + if "identifier" in manifest_node and use_identifiers: name = manifest_node["identifier"] - if manifest_node.get("alias") is not None: + if ( + manifest_node.get("alias") is not None + and manifest_node.get("resource_type") + != "test" # tests have non-human-friendly aliases, so we don't want to use it for tests + ): name = manifest_node["alias"] if not node_name_pattern.allowed(key): @@ -350,10 +525,11 @@ def extract_dbt_entities( catalog_type = None if catalog_node is None: - report.report_warning( - key, - f"Entity {key} ({name}) is in manifest but missing from catalog", - ) + if materialization != "test": + report.report_warning( + key, + f"Entity {key} ({name}) is in manifest but missing from catalog", + ) else: catalog_type = all_catalog_entities[key]["metadata"]["type"] @@ -372,9 +548,11 @@ def extract_dbt_entities( meta_props = manifest_node.get("config", {}).get("meta", {}) dbtNode = DBTNode( dbt_name=key, + dbt_adapter=manifest_adapter, database=manifest_node["database"], schema=manifest_node["schema"], name=name, + alias=manifest_node.get("alias"), dbt_file_path=manifest_node["original_file_path"], node_type=manifest_node["resource_type"], max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"), @@ -389,12 +567,14 @@ def extract_dbt_entities( query_tag=query_tag_props, tags=tags, owner=owner, + compiled_sql=manifest_node.get("compiled_sql"), ) # overwrite columns from catalog - if ( - dbtNode.materialization != "ephemeral" - ): # we don't want columns if platform isn't 'dbt' + if dbtNode.materialization not in [ + "ephemeral", + "test", + ]: # we don't want columns if platform isn't 'dbt' logger.debug("Loading schema info") catalog_node = all_catalog_entities.get(key) @@ -431,9 +611,12 @@ def get_urn_from_dbtNode( data_platform_instance: Optional[str], ) -> str: db_fqn = get_db_fqn(database, schema, name) - if data_platform_instance is not None and target_platform == DBT_PLATFORM: - db_fqn = f"{data_platform_instance}.{db_fqn}" - return mce_builder.make_dataset_urn(target_platform, db_fqn, env) + return mce_builder.make_dataset_urn_with_platform_instance( + platform=target_platform, + name=db_fqn, + platform_instance=data_platform_instance, + env=env, + ) def get_custom_properties(node: DBTNode) -> Dict[str, str]: @@ -460,9 +643,11 @@ def get_upstreams( all_nodes: Dict[str, Dict[str, Any]], use_identifiers: bool, target_platform: str, + target_platform_instance: Optional[str], environment: str, disable_dbt_node_creation: bool, platform_instance: Optional[str], + legacy_skip_source_lineage: Optional[bool], ) -> List[str]: upstream_urns = [] @@ -488,18 +673,22 @@ def get_upstreams( # create lineages for platform nodes otherwise, for dbt node, we connect it to another dbt node or a platform # node. platform_value = DBT_PLATFORM + platform_instance_value = platform_instance if disable_dbt_node_creation: + # we generate all urns in the target platform universe platform_value = target_platform + platform_instance_value = target_platform_instance else: materialized = upstream_manifest_node.get("config", {}).get("materialized") resource_type = upstream_manifest_node["resource_type"] - if ( - materialized in {"view", "table", "incremental"} - or resource_type == "source" + if materialized in {"view", "table", "incremental"} or ( + resource_type == "source" and legacy_skip_source_lineage ): + # upstream urns point to the target platform platform_value = target_platform + platform_instance_value = target_platform_instance upstream_urns.append( get_urn_from_dbtNode( @@ -508,7 +697,7 @@ def get_upstreams( name, platform_value, environment, - platform_instance, + platform_instance_value, ) ) return upstream_urns @@ -543,11 +732,12 @@ def get_upstream_lineage(upstream_urns: List[str]) -> UpstreamLineage: **SNOWFLAKE_TYPES_MAP, **BIGQUERY_TYPES_MAP, **SPARK_SQL_TYPES_MAP, + **TRINO_SQL_TYPES_MAP, } def get_column_type( - report: DBTSourceReport, dataset_name: str, column_type: str + report: DBTSourceReport, dataset_name: str, column_type: str, dbt_adapter: str ) -> SchemaFieldDataType: """ Maps known DBT types to datahub types @@ -555,8 +745,11 @@ def get_column_type( TypeClass: Any = _field_type_mapping.get(column_type) if TypeClass is None: - # attempt Postgres modified type - TypeClass = resolve_postgres_modified_type(column_type) + # resolve modified type + if dbt_adapter == "trino": + TypeClass = resolve_trino_modified_type(column_type) + elif dbt_adapter == "postgres": + TypeClass = resolve_postgres_modified_type(column_type) # if still not found, report the warning if TypeClass is None: @@ -598,7 +791,9 @@ def get_schema_metadata( field = SchemaField( fieldPath=column.name, nativeDataType=column.data_type, - type=get_column_type(report, node.dbt_name, column.data_type), + type=get_column_type( + report, node.dbt_name, column.data_type, node.dbt_adapter + ), description=description, nullable=False, # TODO: actually autodetect this recursive=False, @@ -626,6 +821,236 @@ def get_schema_metadata( ) +@dataclass +class AssertionParams: + scope: Union[DatasetAssertionScopeClass, str] + operator: Union[AssertionStdOperatorClass, str] + aggregation: Union[AssertionStdAggregationClass, str] + parameters: Optional[Callable[[Dict[str, str]], AssertionStdParametersClass]] = None + logic_fn: Optional[Callable[[Dict[str, str]], Optional[str]]] = None + + +def _get_name_for_relationship_test(kw_args: Dict[str, str]) -> Optional[str]: + """ + Try to produce a useful string for the name of a relationship constraint. + Return None if we fail to + """ + destination_ref = kw_args.get("to") + source_ref = kw_args.get("model") + column_name = kw_args.get("column_name") + dest_field_name = kw_args.get("field") + if not destination_ref or not source_ref or not column_name or not dest_field_name: + # base assertions are violated, bail early + return None + m = re.match(r"^ref\(\'(.*)\'\)$", destination_ref) + if m: + destination_table = m.group(1) + else: + destination_table = destination_ref + m = re.search(r"ref\(\'(.*)\'\)", source_ref) + if m: + source_table = m.group(1) + else: + source_table = source_ref + return f"{source_table}.{column_name} referential integrity to {destination_table}.{dest_field_name}" + + +class DBTTestStep(BaseModel): + name: Optional[str] = None + started_at: Optional[str] = None + completed_at: Optional[str] = None + + +class DBTTestResult(BaseModel): + class Config: + extra = "allow" + + status: str + timing: List[DBTTestStep] = [] + unique_id: str + failures: Optional[int] = None + message: Optional[str] = None + + +class DBTRunMetadata(BaseModel): + dbt_schema_version: str + dbt_version: str + generated_at: str + invocation_id: str + + +class DBTTest: + + test_name_to_assertion_map = { + "not_null": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass.NOT_NULL, + aggregation=AssertionStdAggregationClass.IDENTITY, + ), + "unique": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass.EQUAL_TO, + aggregation=AssertionStdAggregationClass.UNIQUE_PROPOTION, + parameters=lambda _: AssertionStdParametersClass( + value=AssertionStdParameterClass( + value="1.0", + type=AssertionStdParameterTypeClass.NUMBER, + ) + ), + ), + "accepted_values": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass.IN, + aggregation=AssertionStdAggregationClass.IDENTITY, + parameters=lambda kw_args: AssertionStdParametersClass( + value=AssertionStdParameterClass( + value=json.dumps(kw_args.get("values")), + type=AssertionStdParameterTypeClass.SET, + ), + ), + ), + "relationships": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass._NATIVE_, + aggregation=AssertionStdAggregationClass.IDENTITY, + parameters=lambda kw_args: AssertionStdParametersClass( + value=AssertionStdParameterClass( + value=json.dumps(kw_args.get("values")), + type=AssertionStdParameterTypeClass.SET, + ), + ), + logic_fn=_get_name_for_relationship_test, + ), + "dbt_expectations.expect_column_values_to_not_be_null": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass.NOT_NULL, + aggregation=AssertionStdAggregationClass.IDENTITY, + ), + "dbt_expectations.expect_column_values_to_be_between": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass.BETWEEN, + aggregation=AssertionStdAggregationClass.IDENTITY, + parameters=lambda x: AssertionStdParametersClass( + minValue=AssertionStdParameterClass( + value=str(x.get("min_value", "unknown")), + type=AssertionStdParameterTypeClass.NUMBER, + ), + maxValue=AssertionStdParameterClass( + value=str(x.get("max_value", "unknown")), + type=AssertionStdParameterTypeClass.NUMBER, + ), + ), + ), + "dbt_expectations.expect_column_values_to_be_in_set": AssertionParams( + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass.IN, + aggregation=AssertionStdAggregationClass.IDENTITY, + parameters=lambda kw_args: AssertionStdParametersClass( + value=AssertionStdParameterClass( + value=json.dumps(kw_args.get("value_set")), + type=AssertionStdParameterTypeClass.SET, + ), + ), + ), + } + + @staticmethod + def load_test_results( + config: DBTConfig, + test_results_json: Dict[str, Any], + test_nodes: List[DBTNode], + manifest_nodes: Dict[str, Any], + ) -> Iterable[MetadataWorkUnit]: + if not config.entities_enabled.can_emit_test_results: + logger.debug("Skipping test result emission since it is turned off.") + return [] + + args = test_results_json.get("args", {}) + dbt_metadata = DBTRunMetadata.parse_obj(test_results_json.get("metadata", {})) + test_nodes_map: Dict[str, DBTNode] = {x.dbt_name: x for x in test_nodes} + if "test" in args.get("which", "") or "test" in args.get("rpc_method", ""): + # this was a test run + results = test_results_json.get("results", []) + for result in results: + try: + test_result = DBTTestResult.parse_obj(result) + id = test_result.unique_id + test_node = test_nodes_map.get(id) + assert test_node, f"Failed to find test_node {id} in the catalog" + upstream_urns = get_upstreams( + test_node.upstream_nodes, + manifest_nodes, + config.use_identifiers, + config.target_platform, + config.target_platform_instance, + config.env, + config.disable_dbt_node_creation, + config.platform_instance, + config.backcompat_skip_source_on_lineage_edge, + ) + assertion_urn = mce_builder.make_assertion_urn( + mce_builder.datahub_guid( + { + "platform": DBT_PLATFORM, + "name": test_result.unique_id, + "instance": config.platform_instance, + } + ) + ) + + if test_result.status != "pass": + native_results = {"message": test_result.message or ""} + if test_result.failures: + native_results.update( + {"failures": str(test_result.failures)} + ) + else: + native_results = {} + + stage_timings = {x.name: x.started_at for x in test_result.timing} + # look for execution start time, fall back to compile start time and finally generation time + execution_timestamp = ( + stage_timings.get("execute") + or stage_timings.get("compile") + or dbt_metadata.generated_at + ) + + execution_timestamp_parsed = datetime.strptime( + execution_timestamp, "%Y-%m-%dT%H:%M:%S.%fZ" + ) + + for upstream in upstream_urns: + assertionResult = AssertionRunEventClass( + timestampMillis=int( + execution_timestamp_parsed.timestamp() * 1000.0 + ), + assertionUrn=assertion_urn, + asserteeUrn=upstream, + runId=dbt_metadata.invocation_id, + result=AssertionResultClass( + type=AssertionResultTypeClass.SUCCESS + if test_result.status == "pass" + else AssertionResultTypeClass.FAILURE, + nativeResults=native_results, + ), + status=AssertionRunStatusClass.COMPLETE, + ) + + event = MetadataChangeProposalWrapper( + entityType="assertion", + entityUrn=assertion_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="assertionRunEvent", + aspect=assertionResult, + ) + yield MetadataWorkUnit( + id=f"{assertion_urn}-assertionRunEvent-{upstream}", + mcp=event, + ) + except Exception as e: + logger.debug(f"Failed to process test result {result} due to {e}") + + @platform_name("dbt") @config_class(DBTConfig) @support_status(SupportStatus.CERTIFIED) @@ -640,7 +1065,7 @@ class DBTSource(StatefulIngestionSourceBase): - dbt Ephemeral: for nodes in the dbt manifest file that are ephemeral models - dbt Sources: for nodes that are sources on top of the underlying platform tables - dbt Seed: for seed entities - - dbt Test: for dbt test entities + - dbt Tests as Assertions: for dbt test entities (starting with version 0.8.38.1) Note: 1. It also generates lineage between the `dbt` nodes (e.g. ephemeral nodes that depend on other dbt sources) as well as lineage between the `dbt` nodes and the underlying (target) platform nodes (e.g. BigQuery Table -> dbt Source, dbt View -> BigQuery View). @@ -649,7 +1074,7 @@ class DBTSource(StatefulIngestionSourceBase): The artifacts used by this source are: - [dbt manifest file](https://docs.getdbt.com/reference/artifacts/manifest-json) - - This file contains model, source and lineage data. + - This file contains model, source, tests and lineage data. - [dbt catalog file](https://docs.getdbt.com/reference/artifacts/catalog-json) - This file contains schema data. - dbt does not record schema data for Ephemeral models, as such datahub will show Ephemeral models in the lineage, however there will be no associated schema for Ephemeral models @@ -657,7 +1082,9 @@ class DBTSource(StatefulIngestionSourceBase): - This file contains metadata for sources with freshness checks. - We transfer dbt's freshness checks to DataHub's last-modified fields. - Note that this file is optional – if not specified, we'll use time of ingestion instead as a proxy for time last-modified. - + - [dbt run_results file](https://docs.getdbt.com/reference/artifacts/run-results-json) + - This file contains metadata from the result of a dbt run, e.g. dbt test + - When provided, we transfer dbt test run results into assertion run events to see a timeline of test runs on the dataset """ @classmethod @@ -676,15 +1103,48 @@ def __init__(self, config: DBTConfig, ctx: PipelineContext, platform: str): self.config.owner_extraction_pattern ) + def get_last_dbt_checkpoint( + self, job_id: JobId, checkpoint_state_class: Type[DbtCheckpointState] + ) -> Optional[Checkpoint]: + + last_checkpoint: Optional[Checkpoint] + is_conversion_required: bool = False + try: + # Best-case that last checkpoint state is DbtCheckpointState + last_checkpoint = self.get_last_checkpoint(job_id, checkpoint_state_class) + except Exception as e: + # Backward compatibility for old dbt ingestion source which was saving dbt-nodes in + # BaseSQLAlchemyCheckpointState + last_checkpoint = self.get_last_checkpoint( + job_id, BaseSQLAlchemyCheckpointState + ) + logger.debug( + f"Found BaseSQLAlchemyCheckpointState as checkpoint state (got {e})." + ) + is_conversion_required = True + + if last_checkpoint is not None and is_conversion_required: + # Map the BaseSQLAlchemyCheckpointState to DbtCheckpointState + dbt_checkpoint_state: DbtCheckpointState = DbtCheckpointState() + dbt_checkpoint_state.encoded_node_urns = ( + cast(BaseSQLAlchemyCheckpointState, last_checkpoint.state) + ).encoded_table_urns + # Old dbt source was not supporting the assertion + dbt_checkpoint_state.encoded_assertion_urns = [] + last_checkpoint.state = dbt_checkpoint_state + + return last_checkpoint + # TODO: Consider refactoring this logic out for use across sources as it is leading to a significant amount of # code duplication. def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: - last_checkpoint = self.get_last_checkpoint( - self.get_default_ingestion_job_id(), BaseSQLAlchemyCheckpointState + last_checkpoint: Optional[Checkpoint] = self.get_last_dbt_checkpoint( + self.get_default_ingestion_job_id(), DbtCheckpointState ) cur_checkpoint = self.get_current_checkpoint( self.get_default_ingestion_job_id() ) + if ( self.config.stateful_ingestion and self.config.stateful_ingestion.remove_stale_metadata @@ -695,7 +1155,7 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: ): logger.debug("Checking for stale entity removal.") - def soft_delete_item(urn: str, type: str) -> Iterable[MetadataWorkUnit]: + def get_soft_delete_item_workunit(urn: str, type: str) -> MetadataWorkUnit: logger.info(f"Soft-deleting stale entity of type {type} - {urn}.") mcp = MetadataChangeProposalWrapper( @@ -708,21 +1168,29 @@ def soft_delete_item(urn: str, type: str) -> Iterable[MetadataWorkUnit]: wu = MetadataWorkUnit(id=f"soft-delete-{type}-{urn}", mcp=mcp) self.report.report_workunit(wu) self.report.report_stale_entity_soft_deleted(urn) - yield wu + return wu - last_checkpoint_state = cast( - BaseSQLAlchemyCheckpointState, last_checkpoint.state - ) - cur_checkpoint_state = cast( - BaseSQLAlchemyCheckpointState, cur_checkpoint.state - ) + last_checkpoint_state = cast(DbtCheckpointState, last_checkpoint.state) + cur_checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) - for table_urn in last_checkpoint_state.get_table_urns_not_in( - cur_checkpoint_state - ): - yield from soft_delete_item(table_urn, "dataset") + urns_to_soft_delete_by_type: Dict = { + "dataset": [ + node_urn + for node_urn in last_checkpoint_state.get_node_urns_not_in( + cur_checkpoint_state + ) + ], + "assertion": [ + assertion_urn + for assertion_urn in last_checkpoint_state.get_assertion_urns_not_in( + cur_checkpoint_state + ) + ], + } + for entity_type in urns_to_soft_delete_by_type: + for urn in urns_to_soft_delete_by_type[entity_type]: + yield get_soft_delete_item_workunit(urn, entity_type) - # s3://data-analysis.pelotime.com/dbt-artifacts/data-engineering-dbt/catalog.json def load_file_as_json(self, uri: str) -> Any: if re.match("^https?://", uri): return json.loads(requests.get(uri).text) @@ -753,6 +1221,7 @@ def loadManifestAndCatalog( Optional[str], Optional[str], Optional[str], + Optional[str], Dict[str, Dict[str, Any]], ]: dbt_manifest_json = self.load_file_as_json(manifest_path) @@ -769,6 +1238,7 @@ def loadManifestAndCatalog( "dbt_schema_version" ) manifest_version = dbt_manifest_json.get("metadata", {}).get("dbt_version") + manifest_adapter = dbt_manifest_json.get("metadata", {}).get("adapter_type") catalog_schema = dbt_catalog_json.get("metadata", {}).get("dbt_schema_version") catalog_version = dbt_catalog_json.get("metadata", {}).get("dbt_version") @@ -787,23 +1257,209 @@ def loadManifestAndCatalog( all_manifest_entities, all_catalog_entities, sources_results, + manifest_adapter, load_schemas, use_identifiers, tag_prefix, node_type_pattern, report, node_name_pattern, + self.config.entities_enabled, ) return ( nodes, manifest_schema, manifest_version, + manifest_adapter, catalog_schema, catalog_version, all_manifest_entities, ) + def create_test_entity_mcps( + self, + test_nodes: List[DBTNode], + custom_props: Dict[str, str], + manifest_nodes: Dict[str, Dict[str, Any]], + ) -> Iterable[MetadataWorkUnit]: + def string_map(input_map: Dict[str, Any]) -> Dict[str, str]: + return {k: str(v) for k, v in input_map.items()} + + if not self.config.entities_enabled.can_emit_node_type("test"): + return [] + + for node in test_nodes: + node_datahub_urn = mce_builder.make_assertion_urn( + mce_builder.datahub_guid( + { + "platform": DBT_PLATFORM, + "name": node.dbt_name, + "instance": self.config.platform_instance, + } + ) + ) + self.save_checkpoint(node_datahub_urn, "assertion") + + dpi_mcp = MetadataChangeProposalWrapper( + entityType="assertion", + entityUrn=node_datahub_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="dataPlatformInstance", + aspect=DataPlatformInstanceClass( + platform=mce_builder.make_data_platform_urn(DBT_PLATFORM) + ), + ) + wu = MetadataWorkUnit( + id=f"{node_datahub_urn}-dataplatformInstance", mcp=dpi_mcp + ) + self.report.report_workunit(wu) + yield wu + + upstream_urns = get_upstreams( + upstreams=node.upstream_nodes, + all_nodes=manifest_nodes, + use_identifiers=self.config.use_identifiers, + target_platform=self.config.target_platform, + target_platform_instance=self.config.target_platform_instance, + environment=self.config.env, + disable_dbt_node_creation=self.config.disable_dbt_node_creation, + platform_instance=None, + legacy_skip_source_lineage=self.config.backcompat_skip_source_on_lineage_edge, + ) + + raw_node = manifest_nodes.get(node.dbt_name) + if raw_node is None: + logger.warning( + f"Failed to find test node {node.dbt_name} in the manifest" + ) + continue + + test_metadata = raw_node.get("test_metadata", {}) + kw_args = test_metadata.get("kwargs", {}) + for upstream_urn in upstream_urns: + qualified_test_name = ( + (test_metadata.get("namespace") or "") + + "." + + (test_metadata.get("name") or "") + ) + qualified_test_name = ( + qualified_test_name[1:] + if qualified_test_name.startswith(".") + else qualified_test_name + ) + + if qualified_test_name in DBTTest.test_name_to_assertion_map: + assertion_params: AssertionParams = ( + DBTTest.test_name_to_assertion_map[qualified_test_name] + ) + assertion_info = AssertionInfoClass( + type=AssertionTypeClass.DATASET, + customProperties=custom_props, + datasetAssertion=DatasetAssertionInfoClass( + dataset=upstream_urn, + scope=assertion_params.scope, + operator=assertion_params.operator, + fields=[ + mce_builder.make_schema_field_urn( + upstream_urn, kw_args.get("column_name") + ) + ] + if assertion_params.scope + == DatasetAssertionScopeClass.DATASET_COLUMN + else [], + nativeType=node.name, + aggregation=assertion_params.aggregation, + parameters=assertion_params.parameters(kw_args) + if assertion_params.parameters + else None, + logic=assertion_params.logic_fn(kw_args) + if assertion_params.logic_fn + else None, + nativeParameters=string_map(kw_args), + ), + ) + elif kw_args.get("column_name"): + # no match with known test types, column-level test + assertion_info = AssertionInfoClass( + type=AssertionTypeClass.DATASET, + customProperties=custom_props, + datasetAssertion=DatasetAssertionInfoClass( + dataset=upstream_urn, + scope=DatasetAssertionScopeClass.DATASET_COLUMN, + operator=AssertionStdOperatorClass._NATIVE_, + fields=[ + mce_builder.make_schema_field_urn( + upstream_urn, kw_args.get("column_name") + ) + ], + nativeType=node.name, + logic=node.compiled_sql + if node.compiled_sql + else node.raw_sql, + aggregation=AssertionStdAggregationClass._NATIVE_, + nativeParameters=string_map(kw_args), + ), + ) + else: + # no match with known test types, default to row-level test + assertion_info = AssertionInfoClass( + type=AssertionTypeClass.DATASET, + customProperties=custom_props, + datasetAssertion=DatasetAssertionInfoClass( + dataset=upstream_urn, + scope=DatasetAssertionScopeClass.DATASET_ROWS, + operator=AssertionStdOperatorClass._NATIVE_, + logic=node.compiled_sql + if node.compiled_sql + else node.raw_sql, + nativeType=node.name, + aggregation=AssertionStdAggregationClass._NATIVE_, + nativeParameters=string_map(kw_args), + ), + ) + wu = MetadataWorkUnit( + id=f"{node_datahub_urn}-assertioninfo", + mcp=MetadataChangeProposalWrapper( + entityType="assertion", + entityUrn=node_datahub_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="assertionInfo", + aspect=assertion_info, + ), + ) + self.report.report_workunit(wu) + yield wu + + if self.config.delete_tests_as_datasets: + mce_platform = ( + self.config.target_platform + if self.config.disable_dbt_node_creation + else DBT_PLATFORM + ) + node_datahub_urn = get_urn_from_dbtNode( + node.database, + node.schema, + node.alias or node.name, # previous code used the alias + mce_platform, + self.config.env, + self.config.platform_instance + if mce_platform == DBT_PLATFORM + else None, + ) + soft_delete_mcp = MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=node_datahub_urn, + aspectName="status", + aspect=StatusClass(removed=True), + ) + soft_delete_wu = MetadataWorkUnit( + id=f"{node_datahub_urn}-status", mcp=soft_delete_mcp + ) + self.report.report_workunit(soft_delete_wu) + yield soft_delete_wu + # create workunits from dbt nodes def get_workunits(self) -> Iterable[MetadataWorkUnit]: if self.config.write_semantics == "PATCH" and not self.ctx.graph: @@ -816,6 +1472,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: nodes, manifest_schema, manifest_version, + manifest_adapter, catalog_schema, catalog_version, manifest_nodes_raw, @@ -834,6 +1491,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: additional_custom_props = { "manifest_schema": manifest_schema, "manifest_version": manifest_version, + "manifest_adapter": manifest_adapter, "catalog_schema": catalog_schema, "catalog_version": catalog_version, } @@ -844,21 +1502,42 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: if value is not None } + non_test_nodes = [ + dataset_node for dataset_node in nodes if dataset_node.node_type != "test" + ] + test_nodes = [test_node for test_node in nodes if test_node.node_type == "test"] + if not self.config.disable_dbt_node_creation: yield from self.create_platform_mces( - nodes, + non_test_nodes, additional_custom_props_filtered, manifest_nodes_raw, DBT_PLATFORM, + self.config.platform_instance, ) yield from self.create_platform_mces( - nodes, + non_test_nodes, additional_custom_props_filtered, manifest_nodes_raw, self.config.target_platform, + self.config.target_platform_instance, + ) + + yield from self.create_test_entity_mcps( + test_nodes, + additional_custom_props_filtered, + manifest_nodes_raw, ) + if self.config.test_results_path: + yield from DBTTest.load_test_results( + self.config, + self.load_file_as_json(self.config.test_results_path), + test_nodes, + manifest_nodes_raw, + ) + if self.is_stateful_ingestion_configured(): # Clean up stale entities. yield from self.gen_removed_entity_workunits() @@ -877,10 +1556,12 @@ def remove_duplicate_urns_from_checkpoint_state(self) -> None: ) if cur_checkpoint is not None: - # Utilizing BaseSQLAlchemyCheckpointState class to save state - checkpoint_state = cast(BaseSQLAlchemyCheckpointState, cur_checkpoint.state) - checkpoint_state.encoded_table_urns = list( - set(checkpoint_state.encoded_table_urns) + checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) + checkpoint_state.encoded_node_urns = list( + set(checkpoint_state.encoded_node_urns) + ) + checkpoint_state.encoded_assertion_urns = list( + set(checkpoint_state.encoded_assertion_urns) ) def create_platform_mces( @@ -889,6 +1570,7 @@ def create_platform_mces( additional_custom_props_filtered: Dict[str, str], manifest_nodes_raw: Dict[str, Dict[str, Any]], mce_platform: str, + mce_platform_instance: Optional[str], ) -> Iterable[MetadataWorkUnit]: """ This function creates mce based out of dbt nodes. Since dbt ingestion creates "dbt" nodes @@ -913,17 +1595,22 @@ def create_platform_mces( "SOURCE_CONTROL", self.config.strip_user_ids_from_email, ) - for node in dbt_nodes: + node_datahub_urn = get_urn_from_dbtNode( node.database, node.schema, node.name, mce_platform, self.config.env, - self.config.platform_instance, + mce_platform_instance, ) - self.save_checkpoint(node_datahub_urn) + if not self.config.entities_enabled.can_emit_node_type(node.node_type): + logger.debug( + f"Skipping emission of node {node_datahub_urn} because node_type {node.node_type} is disabled" + ) + continue + self.save_checkpoint(node_datahub_urn, "dataset") meta_aspects: Dict[str, Any] = {} if self.config.enable_meta_mapping and node.meta: @@ -999,18 +1686,21 @@ def create_platform_mces( self.report.report_workunit(wu) yield wu - def save_checkpoint(self, node_datahub_urn: str) -> None: - if self.is_stateful_ingestion_configured(): - cur_checkpoint = self.get_current_checkpoint( - self.get_default_ingestion_job_id() - ) + def save_checkpoint(self, urn: str, entity_type: str) -> None: + # if stateful ingestion is not configured then return + if not self.is_stateful_ingestion_configured(): + return - if cur_checkpoint is not None: - # Utilizing BaseSQLAlchemyCheckpointState class to save state - checkpoint_state = cast( - BaseSQLAlchemyCheckpointState, cur_checkpoint.state - ) - checkpoint_state.add_table_urn(node_datahub_urn) + cur_checkpoint = self.get_current_checkpoint( + self.get_default_ingestion_job_id() + ) + # if no checkpoint found then return + if cur_checkpoint is None: + return + + # Cast and set the state + checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) + checkpoint_state.set_checkpoint_urn(urn, entity_type) def extract_query_tag_aspects( self, @@ -1033,25 +1723,6 @@ def get_aspect_from_dataset( return aspect return None - # TODO: Remove. keeping this till PR review - # def get_owners_from_dataset_snapshot(self, dataset_snapshot: DatasetSnapshot) -> Optional[OwnershipClass]: - # for aspect in dataset_snapshot.aspects: - # if isinstance(aspect, OwnershipClass): - # return aspect - # return None - # - # def get_tag_aspect_from_dataset_snapshot(self, dataset_snapshot: DatasetSnapshot) -> Optional[GlobalTagsClass]: - # for aspect in dataset_snapshot.aspects: - # if isinstance(aspect, GlobalTagsClass): - # return aspect - # return None - # - # def get_term_aspect_from_dataset_snapshot(self, dataset_snapshot: DatasetSnapshot) -> Optional[GlossaryTermsClass]: - # for aspect in dataset_snapshot.aspects: - # if isinstance(aspect, GlossaryTermsClass): - # return aspect - # return None - def get_patched_mce(self, mce): owner_aspect = self.get_aspect_from_dataset( mce.proposedSnapshot, OwnershipClass @@ -1183,7 +1854,10 @@ def _aggregate_owners( self, node: DBTNode, meta_owner_aspects: Any ) -> List[OwnerClass]: owner_list: List[OwnerClass] = [] - if node.owner: + if meta_owner_aspects and self.config.enable_meta_mapping: + # we disregard owners generated from node.owner because that is also coming from the meta section + owner_list = meta_owner_aspects.owners + elif node.owner: owner: str = node.owner if self.compiled_owner_extraction_pattern: match: Optional[Any] = re.match( @@ -1205,9 +1879,6 @@ def _aggregate_owners( ) ) - if meta_owner_aspects and self.config.enable_meta_mapping: - owner_list.extend(meta_owner_aspects.owners) - owner_list = sorted(owner_list, key=lambda x: x.owner) return owner_list @@ -1259,9 +1930,11 @@ def _create_lineage_aspect_for_dbt_node( manifest_nodes_raw, self.config.use_identifiers, self.config.target_platform, + self.config.target_platform_instance, self.config.env, self.config.disable_dbt_node_creation, self.config.platform_instance, + self.config.backcompat_skip_source_on_lineage_edge, ) # if a node is of type source in dbt, its upstream lineage should have the corresponding table/view @@ -1274,7 +1947,7 @@ def _create_lineage_aspect_for_dbt_node( node.name, self.config.target_platform, self.config.env, - self.config.platform_instance, + self.config.target_platform_instance, ) ) if upstream_urns: @@ -1293,9 +1966,11 @@ def _create_lineage_aspect_for_platform_node( manifest_nodes_raw, self.config.use_identifiers, self.config.target_platform, + self.config.target_platform_instance, self.config.env, self.config.disable_dbt_node_creation, - None, + self.config.platform_instance, + self.config.backcompat_skip_source_on_lineage_edge, ) if upstream_urns: return get_upstream_lineage(upstream_urns) @@ -1380,8 +2055,7 @@ def create_checkpoint(self, job_id: JobId) -> Optional[Checkpoint]: platform_instance_id=self.get_platform_instance_id(), run_id=self.ctx.run_id, config=self.config, - # Reusing BaseSQLAlchemyCheckpointState as it has needed functionality to support statefulness of DBT - state=BaseSQLAlchemyCheckpointState(), + state=DbtCheckpointState(), ) return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/__init__.py new file mode 100644 index 00000000000000..2d2a70707d354e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/__init__.py @@ -0,0 +1 @@ +from datahub.ingestion.source.delta_lake.source import DeltaLakeSource diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py new file mode 100644 index 00000000000000..043facca1ebb62 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py @@ -0,0 +1,96 @@ +import logging +from typing import Any, Dict, Optional + +import pydantic +from pydantic import Field + +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.source_common import ( + ConfigModel, + EnvBasedSourceConfigBase, + PlatformSourceConfigBase, +) +from datahub.ingestion.source.aws.aws_common import AwsSourceConfig +from datahub.ingestion.source.aws.s3_util import is_s3_uri + +# hide annoying debug errors from py4j +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + + +class S3(ConfigModel): + aws_config: AwsSourceConfig = Field(default=None, description="AWS configuration") + + # Whether or not to create in datahub from the s3 bucket + use_s3_bucket_tags: Optional[bool] = Field( + False, description="Whether or not to create tags in datahub from the s3 bucket" + ) + # Whether or not to create in datahub from the s3 object + use_s3_object_tags: Optional[bool] = Field( + False, + description="# Whether or not to create tags in datahub from the s3 object", + ) + + +class DeltaLakeSourceConfig(PlatformSourceConfigBase, EnvBasedSourceConfigBase): + class Config: + arbitrary_types_allowed = True + + base_path: str = Field( + description="Path to table (s3 or local file system). If path is not a delta table path " + "then all subfolders will be scanned to detect and ingest delta tables." + ) + relative_path: str = Field( + default=None, + description="If set, delta-tables will be searched at location " + "'/' and URNs will be created using " + "relative_path only.", + ) + platform: str = Field( + default="delta-lake", + description="The platform that this source connects to", + const=True, + ) + platform_instance: Optional[str] = Field( + default=None, + description="The instance of the platform that all assets produced by this recipe belong to", + ) + table_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="regex patterns for tables to filter in ingestion.", + ) + version_history_lookback: Optional[int] = Field( + default=1, + description="Number of previous version histories to be ingested. Defaults to 1. If set to -1 all version history will be ingested.", + ) + + s3: Optional[S3] = Field() + + # to be set internally + _is_s3: bool + _complete_path: str + + def is_s3(self): + return self._is_s3 + + def get_complete_path(self): + return self._complete_path + + @pydantic.validator("version_history_lookback") + def negative_version_history_implies_no_limit(cls, v): + if v and v < 0: + return None + return v + + @pydantic.root_validator() + def validate_config(cls, values: Dict) -> Dict[str, Any]: + values["_is_s3"] = is_s3_uri(values.get("base_path", "")) + if values["_is_s3"]: + if values.get("s3") is None: + raise ValueError("s3 config must be set for s3 path") + values["_complete_path"] = values.get("base_path") + if values.get("relative_path") is not None: + values[ + "_complete_path" + ] = f"{values['_complete_path'].rstrip('/')}/{values['relative_path'].lstrip('/')}" + return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py new file mode 100644 index 00000000000000..32f22b9d84773e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py @@ -0,0 +1,32 @@ +from typing import Optional + +from deltalake import DeltaTable, PyDeltaTableError + +from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig + + +def read_delta_table( + path: str, delta_lake_config: DeltaLakeSourceConfig +) -> Optional[DeltaTable]: + delta_table = None + try: + opts = {} + if delta_lake_config.is_s3(): + if delta_lake_config.s3 is None: + raise ValueError("aws_config not set. Cannot browse s3") + if delta_lake_config.s3.aws_config is None: + raise ValueError("aws_config not set. Cannot browse s3") + opts = { + "AWS_ACCESS_KEY_ID": delta_lake_config.s3.aws_config.aws_access_key_id, + "AWS_SECRET_ACCESS_KEY": delta_lake_config.s3.aws_config.aws_secret_access_key, + } + delta_table = DeltaTable(path, storage_options=opts) + + except PyDeltaTableError as e: + if "Not a Delta table" not in str(e): + raise e + return delta_table + + +def get_file_count(delta_table: DeltaTable) -> int: + return len(delta_table.files()) diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/report.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/report.py new file mode 100644 index 00000000000000..e8690a24f8b85e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/report.py @@ -0,0 +1,17 @@ +import dataclasses +from dataclasses import field as dataclass_field +from typing import List + +from datahub.ingestion.api.source import SourceReport + + +@dataclasses.dataclass +class DeltaLakeSourceReport(SourceReport): + files_scanned = 0 + filtered: List[str] = dataclass_field(default_factory=list) + + def report_file_scanned(self) -> None: + self.files_scanned += 1 + + def report_file_dropped(self, file: str) -> None: + self.filtered.append(file) diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py new file mode 100644 index 00000000000000..e32ba5d5c3bb3a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -0,0 +1,340 @@ +import logging +import os +import time +from datetime import datetime +from typing import Callable, Iterable, List + +from deltalake import DeltaTable + +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataset_urn_with_platform_instance, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import PipelineContext, WorkUnit +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.aws.s3_boto_utils import get_s3_tags, list_folders_path +from datahub.ingestion.source.aws.s3_util import ( + get_bucket_name, + get_key_prefix, + strip_s3_prefix, +) +from datahub.ingestion.source.data_lake.data_lake_utils import ContainerWUCreator +from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig +from datahub.ingestion.source.delta_lake.delta_lake_utils import ( + get_file_count, + read_delta_table, +) +from datahub.ingestion.source.delta_lake.report import DeltaLakeSourceReport +from datahub.ingestion.source.schema_inference.csv_tsv import tableschema_type_map +from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot +from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + SchemaField, + SchemaFieldDataType, + SchemaMetadata, +) +from datahub.metadata.schema_classes import ( + ChangeTypeClass, + DatasetPropertiesClass, + NullTypeClass, + OperationClass, + OperationTypeClass, + OtherSchemaClass, +) +from datahub.telemetry import telemetry + +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + +config_options_to_report = [ + "platform", +] + +OPERATION_STATEMENT_TYPES = { + "INSERT": OperationTypeClass.INSERT, + "UPDATE": OperationTypeClass.UPDATE, + "DELETE": OperationTypeClass.DELETE, + "MERGE": OperationTypeClass.UPDATE, + "CREATE": OperationTypeClass.CREATE, + "CREATE_TABLE_AS_SELECT": OperationTypeClass.CREATE, + "CREATE_SCHEMA": OperationTypeClass.CREATE, + "DROP_TABLE": OperationTypeClass.DROP, + "REPLACE TABLE AS SELECT": OperationTypeClass.UPDATE, + "COPY INTO": OperationTypeClass.UPDATE, +} + + +@platform_name("Delta Lake", id="delta-lake") +@config_class(DeltaLakeSourceConfig) +@support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.TAGS, "Can extract S3 object/bucket tags if enabled") +class DeltaLakeSource(Source): + """ + This plugin extracts: + - Column types and schema associated with each delta table + - Custom properties: number_of_files, partition_columns, table_creation_time, location, version etc. + + :::caution + + If you are ingesting datasets from AWS S3, we recommend running the ingestion on a server in the same region to avoid high egress costs. + + ::: + + """ + + source_config: DeltaLakeSourceConfig + report: DeltaLakeSourceReport + profiling_times_taken: List[float] + container_WU_creator: ContainerWUCreator + + def __init__(self, config: DeltaLakeSourceConfig, ctx: PipelineContext): + super().__init__(ctx) + self.source_config = config + self.report = DeltaLakeSourceReport() + # self.profiling_times_taken = [] + config_report = { + config_option: config.dict().get(config_option) + for config_option in config_options_to_report + } + config_report = config_report + + telemetry.telemetry_instance.ping( + "delta_lake_config", + config_report, + ) + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> "Source": + config = DeltaLakeSourceConfig.parse_obj(config_dict) + return cls(config, ctx) + + def get_fields(self, delta_table: DeltaTable) -> List[SchemaField]: + + fields: List[SchemaField] = [] + + for raw_field in delta_table.schema().fields: + field = SchemaField( + fieldPath=raw_field.name, + type=SchemaFieldDataType( + tableschema_type_map.get(raw_field.type.type, NullTypeClass)() + ), + nativeDataType=raw_field.type.type, + recursive=False, + nullable=raw_field.nullable, + description=str(raw_field.metadata), + isPartitioningKey=True + if raw_field.name in delta_table.metadata().partition_columns + else False, + ) + fields.append(field) + fields = sorted(fields, key=lambda f: f.fieldPath) + + return fields + + def _create_operation_aspect_wu( + self, delta_table: DeltaTable, dataset_urn: str + ) -> Iterable[MetadataWorkUnit]: + for hist in delta_table.history( + limit=self.source_config.version_history_lookback + ): + + # History schema picked up from https://docs.delta.io/latest/delta-utility.html#retrieve-delta-table-history + reported_time: int = int(time.time() * 1000) + last_updated_timestamp: int = hist["timestamp"] + statement_type = OPERATION_STATEMENT_TYPES.get( + hist.get("operation"), OperationTypeClass.CUSTOM + ) + custom_type = ( + hist.get("operation") + if statement_type == OperationTypeClass.CUSTOM + else None + ) + + operation_custom_properties = dict() + for key, val in hist.items(): + if val is not None: + if isinstance(val, dict): + for k, v in val: + if v is not None: + operation_custom_properties[f"{key}_{k}"] = str(v) + else: + operation_custom_properties[key] = str(val) + operation_custom_properties.pop("timestamp", None) + operation_custom_properties.pop("operation", None) + operation_aspect = OperationClass( + timestampMillis=reported_time, + lastUpdatedTimestamp=last_updated_timestamp, + operationType=statement_type, + customOperationType=custom_type, + customProperties=operation_custom_properties, + ) + + mcp = MetadataChangeProposalWrapper( + entityType="dataset", + aspectName="operation", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_urn, + aspect=operation_aspect, + ) + operational_wu = MetadataWorkUnit( + id=f"{datetime.fromtimestamp(last_updated_timestamp / 1000).isoformat()}-operation-aspect-{dataset_urn}", + mcp=mcp, + ) + self.report.report_workunit(operational_wu) + yield operational_wu + + def ingest_table( + self, delta_table: DeltaTable, path: str + ) -> Iterable[MetadataWorkUnit]: + table_name = ( + delta_table.metadata().name + if delta_table.metadata().name + else path.split("/")[-1] + ) + if not self.source_config.table_pattern.allowed(table_name): + logger.debug( + f"Skipping table ({table_name}) present at location {path} as table pattern does not match" + ) + + logger.debug(f"Ingesting table {table_name} from location {path}") + if self.source_config.relative_path is None: + browse_path: str = ( + strip_s3_prefix(path) if self.source_config.is_s3() else path.strip("/") + ) + else: + browse_path = path.split(self.source_config.base_path)[1].strip("/") + + data_platform_urn = make_data_platform_urn(self.source_config.platform) + logger.info(f"Creating dataset urn with name: {browse_path}") + dataset_urn = make_dataset_urn_with_platform_instance( + self.source_config.platform, + browse_path, + self.source_config.platform_instance, + self.source_config.env, + ) + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[], + ) + + customProperties = { + "number_of_files": str(get_file_count(delta_table)), + "partition_columns": str(delta_table.metadata().partition_columns), + "table_creation_time": str(delta_table.metadata().created_time), + "id": str(delta_table.metadata().id), + "version": str(delta_table.version()), + "location": self.source_config.get_complete_path(), + } + + dataset_properties = DatasetPropertiesClass( + description=delta_table.metadata().description, + name=table_name, + customProperties=customProperties, + ) + dataset_snapshot.aspects.append(dataset_properties) + + fields = self.get_fields(delta_table) + schema_metadata = SchemaMetadata( + schemaName=table_name, + platform=data_platform_urn, + version=delta_table.version(), + hash="", + fields=fields, + platformSchema=OtherSchemaClass(rawSchema=""), + ) + dataset_snapshot.aspects.append(schema_metadata) + + if ( + self.source_config.is_s3() + and self.source_config.s3 + and ( + self.source_config.s3.use_s3_bucket_tags + or self.source_config.s3.use_s3_object_tags + ) + ): + bucket = get_bucket_name(path) + key_prefix = get_key_prefix(path) + s3_tags = get_s3_tags( + bucket, + key_prefix, + dataset_urn, + self.source_config.s3.aws_config, + self.ctx, + self.source_config.s3.use_s3_bucket_tags, + self.source_config.s3.use_s3_object_tags, + ) + if s3_tags is not None: + dataset_snapshot.aspects.append(s3_tags) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + wu = MetadataWorkUnit(id=delta_table.metadata().id, mce=mce) + self.report.report_workunit(wu) + yield wu + + container_wus = self.container_WU_creator.create_container_hierarchy( + browse_path, self.source_config.is_s3(), dataset_urn + ) + for wu in container_wus: + self.report.report_workunit(wu) + yield wu + + yield from self._create_operation_aspect_wu(delta_table, dataset_urn) + + def process_folder( + self, path: str, get_folders: Callable[[str], Iterable[str]] + ) -> Iterable[MetadataWorkUnit]: + logger.debug(f"Processing folder: {path}") + delta_table = read_delta_table(path, self.source_config) + if delta_table: + logger.debug(f"Delta table found at: {path}") + for wu in self.ingest_table(delta_table, path): + yield wu + else: + for folder in get_folders(path): + yield from self.process_folder(path + "/" + folder, get_folders) + + def s3_get_folders(self, path: str) -> Iterable[str]: + if self.source_config.s3 is not None: + yield from list_folders_path(path, self.source_config.s3.aws_config) + + def local_get_folders(self, path: str) -> Iterable[str]: + if not os.path.isdir(path): + raise Exception( + f"{path} does not exist. Please check base_path configuration." + ) + for _, folders, _ in os.walk(path): + for folder in folders: + yield folder + break + return + + def get_workunits(self) -> Iterable[WorkUnit]: + self.container_WU_creator = ContainerWUCreator( + self.source_config.platform, + self.source_config.platform_instance, + self.source_config.env, + ) + get_folders = ( + self.s3_get_folders + if self.source_config.is_s3() + else self.local_get_folders + ) + for wu in self.process_folder( + self.source_config.get_complete_path(), get_folders + ): + yield wu + + def get_report(self) -> SourceReport: + return self.report + + def close(self): + pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py index c5a6c13bf695df..938628b72769ee 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py +++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py @@ -85,10 +85,12 @@ class ElasticToSchemaFieldConverter: "match_only_text": StringTypeClass, "completion": StringTypeClass, "search_as_you_type": StringTypeClass, + "ip": StringTypeClass, # Records "object": RecordTypeClass, "flattened": RecordTypeClass, "nested": RecordTypeClass, + "geo_point": RecordTypeClass, # Arrays "histogram": ArrayTypeClass, "aggregate_metric_double": ArrayTypeClass, @@ -224,6 +226,13 @@ class ElasticsearchSourceConfig(DatasetSourceConfigBase): default=AllowDenyPattern(allow=[".*"], deny=["^_.*", "^ilm-history.*"]), description="regex patterns for indexes to filter in ingestion.", ) + ingest_index_templates: bool = Field( + default=False, description="Ingests ES index templates if enabled." + ) + index_template_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern(allow=[".*"], deny=["^_.*"]), + description="The regex patterns for filtering index templates to ingest.", + ) @validator("host") def host_colon_port_comma(cls, host_val: str) -> str: @@ -255,9 +264,7 @@ def host_colon_port_comma(cls, host_val: str) -> str: @property def http_auth(self) -> Optional[Tuple[str, str]]: - if self.username is None: - return None - return self.username, self.password or "" + return None if self.username is None else (self.username, self.password or "") @platform_name("Elastic Search") @@ -306,7 +313,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.report.report_index_scanned(index) if self.source_config.index_pattern.allowed(index): - for mcp in self._extract_mcps(index): + for mcp in self._extract_mcps(index, is_index=True): wu = MetadataWorkUnit(id=f"index-{index}", mcp=mcp) self.report.report_workunit(wu) yield wu @@ -317,6 +324,14 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: wu = MetadataWorkUnit(id=f"index-{index}", mcp=mcp) self.report.report_workunit(wu) yield wu + if self.source_config.ingest_index_templates: + templates = self.client.indices.get_template() + for template in templates: + if self.source_config.index_template_pattern.allowed(template): + for mcp in self._extract_mcps(template, is_index=False): + wu = MetadataWorkUnit(id=f"template-{template}", mcp=mcp) + self.report.report_workunit(wu) + yield wu def _get_data_stream_index_count_mcps( self, @@ -338,19 +353,26 @@ def _get_data_stream_index_count_mcps( changeType=ChangeTypeClass.UPSERT, ) - def _extract_mcps(self, index: str) -> Iterable[MetadataChangeProposalWrapper]: - logger.debug(f"index = {index}") - raw_index = self.client.indices.get(index=index) - raw_index_metadata = raw_index[index] - - # 0. Dedup data_streams. - data_stream = raw_index_metadata.get("data_stream") - if data_stream: - index = data_stream - self.data_stream_partition_count[index] += 1 - if self.data_stream_partition_count[index] > 1: - # This is a duplicate, skip processing it further. - return + def _extract_mcps( + self, index: str, is_index: bool = True + ) -> Iterable[MetadataChangeProposalWrapper]: + logger.debug(f"index='{index}', is_index={is_index}") + + if is_index: + raw_index = self.client.indices.get(index=index) + raw_index_metadata = raw_index[index] + + # 0. Dedup data_streams. + data_stream = raw_index_metadata.get("data_stream") + if data_stream: + index = data_stream + self.data_stream_partition_count[index] += 1 + if self.data_stream_partition_count[index] > 1: + # This is a duplicate, skip processing it further. + return + else: + raw_index = self.client.indices.get_template(name=index) + raw_index_metadata = raw_index[index] # 1. Construct and emit the schemaMetadata aspect # 1.1 Generate the schema fields from ES mappings. @@ -403,23 +425,47 @@ def _extract_mcps(self, index: str) -> Iterable[MetadataChangeProposalWrapper]: entityUrn=dataset_urn, aspectName="subTypes", aspect=SubTypesClass( - typeNames=["Index" if not data_stream else "DataStream"] + typeNames=[ + "Index Template" + if not is_index + else "Index" + if not data_stream + else "Datastream" + ] ), changeType=ChangeTypeClass.UPSERT, ) - # 4. Construct and emit properties if needed - index_aliases = raw_index_metadata.get("aliases", {}).keys() + # 4. Construct and emit properties if needed. Will attempt to get the following properties + custom_properties: Dict[str, str] = {} + # 4.1 aliases + index_aliases: List[str] = raw_index_metadata.get("aliases", {}).keys() if index_aliases: - yield MetadataChangeProposalWrapper( - entityType="dataset", - entityUrn=dataset_urn, - aspectName="datasetProperties", - aspect=DatasetPropertiesClass( - customProperties={"aliases": ",".join(index_aliases)} - ), - changeType=ChangeTypeClass.UPSERT, - ) + custom_properties["aliases"] = ",".join(index_aliases) + # 4.2 index_patterns + index_patterns: List[str] = raw_index_metadata.get("index_patterns", []) + if index_patterns: + custom_properties["index_patterns"] = ",".join(index_patterns) + + # 4.3 number_of_shards + index_settings: Dict[str, Any] = raw_index_metadata.get("settings", {}).get( + "index", {} + ) + num_shards: str = index_settings.get("number_of_shards", "") + if num_shards: + custom_properties["num_shards"] = num_shards + # 4.4 number_of_replicas + num_replicas: str = index_settings.get("number_of_replicas", "") + if num_replicas: + custom_properties["num_replicas"] = num_replicas + + yield MetadataChangeProposalWrapper( + entityType="dataset", + entityUrn=dataset_urn, + aspectName="datasetProperties", + aspect=DatasetPropertiesClass(customProperties=custom_properties), + changeType=ChangeTypeClass.UPSERT, + ) # 5. Construct and emit platform instance aspect if self.source_config.platform_instance: diff --git a/metadata-ingestion/src/datahub/ingestion/source/feast.py b/metadata-ingestion/src/datahub/ingestion/source/feast.py index c17ae5c14a85fd..7d4f5360ad4202 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/feast.py +++ b/metadata-ingestion/src/datahub/ingestion/source/feast.py @@ -4,23 +4,23 @@ from pydantic import Field -if sys.version_info >= (3, 7): - from feast import ( - BigQuerySource, - Entity, - Feature, - FeatureStore, - FeatureView, - FileSource, - KafkaSource, - KinesisSource, - OnDemandFeatureView, - ValueType, - ) - from feast.data_source import DataSource, RequestDataSource -else: +if sys.version_info < (3, 7): raise ModuleNotFoundError("The feast plugin requires Python 3.7 or newer.") +from feast import ( + BigQuerySource, + Entity, + Feature, + FeatureStore, + FeatureView, + FileSource, + KafkaSource, + KinesisSource, + OnDemandFeatureView, + ValueType, +) +from feast.data_source import DataSource, RequestDataSource + import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigModel from datahub.emitter.mce_builder import DEFAULT_ENV @@ -52,6 +52,7 @@ assert sys.version_info >= (3, 7) # needed for mypy +# FIXME: ValueType module cannot be used as a type _field_type_mapping: Dict[ValueType, str] = { ValueType.UNKNOWN: MLFeatureDataType.UNKNOWN, ValueType.BYTES: MLFeatureDataType.BYTE, @@ -218,6 +219,7 @@ def _get_entity_workunit( def _get_feature_workunit( self, + # FIXME: FeatureView and OnDemandFeatureView cannot be used as a type feature_view: Union[FeatureView, OnDemandFeatureView], feature: Feature, ) -> MetadataWorkUnit: diff --git a/metadata-ingestion/src/datahub/ingestion/source/file.py b/metadata-ingestion/src/datahub/ingestion/source/file.py index d6cff3d43c8394..f8c5fcaef32249 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/file.py +++ b/metadata-ingestion/src/datahub/ingestion/source/file.py @@ -1,4 +1,5 @@ import json +import os.path from dataclasses import dataclass, field from typing import Iterable, Iterator, Union @@ -11,7 +12,12 @@ platform_name, support_status, ) -from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.source import ( + CapabilityReport, + SourceReport, + TestableSource, + TestConnectionReport, +) from datahub.ingestion.api.workunit import MetadataWorkUnit, UsageStatsWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.mxe import ( MetadataChangeEvent, @@ -60,7 +66,7 @@ class FileSourceConfig(ConfigModel): @config_class(FileSourceConfig) @support_status(SupportStatus.CERTIFIED) @dataclass -class GenericFileSource(Source): +class GenericFileSource(TestableSource): """ This plugin pulls metadata from a previously generated file. The [file sink](../../../../metadata-ingestion/sink_docs/file.md) can produce such files, and a number of samples are included in the [examples/mce_files](../../../../metadata-ingestion/examples/mce_files) directory. """ @@ -90,3 +96,29 @@ def get_report(self): def close(self): pass + + @staticmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + config = FileSourceConfig.parse_obj(config_dict) + is_file = os.path.isfile(config.filename) + readable = os.access(config.filename, os.R_OK) + if is_file and readable: + return TestConnectionReport( + basic_connectivity=CapabilityReport(capable=True) + ) + elif not is_file: + return TestConnectionReport( + basic_connectivity=CapabilityReport( + capable=False, + failure_reason=f"{config.filename} doesn't appear to be a valid file.", + ) + ) + elif not readable: + return TestConnectionReport( + basic_connectivity=CapabilityReport( + capable=False, failure_reason=f"Cannot read file {config.filename}" + ) + ) + else: + # not expected to be here + raise Exception("Not expected to be here.") diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index 781585c2a72ca4..e3e081d65bcf9c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -11,6 +11,7 @@ from great_expectations import __version__ as ge_version +from datahub.configuration.common import ConfigurationError from datahub.telemetry import stats, telemetry # Fun compatibility hack! GE version 0.13.44 broke compatibility with SQLAlchemy 1.3.24. @@ -225,10 +226,9 @@ def _is_single_row_query_method(query: Any) -> bool: return False -# mypy does not yet support ParamSpec. See https://github.com/python/mypy/issues/8645. def _run_with_query_combiner( - method: Callable[Concatenate["_SingleDatasetProfiler", P], None] # type: ignore -) -> Callable[Concatenate["_SingleDatasetProfiler", P], None]: # type: ignore + method: Callable[Concatenate["_SingleDatasetProfiler", P], None] +) -> Callable[Concatenate["_SingleDatasetProfiler", P], None]: @functools.wraps(method) def inner( self: "_SingleDatasetProfiler", *args: P.args, **kwargs: P.kwargs @@ -731,6 +731,7 @@ def generate_profiles( requests: List[GEProfilerRequest], max_workers: int, platform: Optional[str] = None, + profiler_args: Optional[Dict] = None, ) -> Iterable[Tuple[GEProfilerRequest, Optional[DatasetProfileClass]]]: with PerfTimer() as timer, concurrent.futures.ThreadPoolExecutor( max_workers=max_workers @@ -758,6 +759,7 @@ def generate_profiles( query_combiner, request, platform=platform, + profiler_args=profiler_args, ) for request in requests ] @@ -817,11 +819,13 @@ def _generate_profile_from_request( query_combiner: SQLAlchemyQueryCombiner, request: GEProfilerRequest, platform: Optional[str] = None, + profiler_args: Optional[Dict] = None, ) -> Tuple[GEProfilerRequest, Optional[DatasetProfileClass]]: return request, self._generate_single_profile( query_combiner=query_combiner, pretty_name=request.pretty_name, platform=platform, + profiler_args=profiler_args, **request.batch_kwargs, ) @@ -844,8 +848,12 @@ def _generate_single_profile( partition: Optional[str] = None, custom_sql: Optional[str] = None, platform: Optional[str] = None, + profiler_args: Optional[Dict] = None, **kwargs: Any, ) -> Optional[DatasetProfileClass]: + logger.debug( + f"Received single profile request for {pretty_name} for {schema}, {table}, {custom_sql}" + ) bigquery_temp_table: Optional[str] = None ge_config = { @@ -858,16 +866,28 @@ def _generate_single_profile( # We have to create temporary tables if offset or limit or custom sql is set on Bigquery if custom_sql or self.config.limit or self.config.offset: + if profiler_args is not None: + temp_table_db = profiler_args.get("temp_table_db", schema) + if platform is not None and platform == "bigquery": + ge_config["schema"] = temp_table_db + if self.config.bigquery_temp_table_schema: - bigquery_temp_table = ( - f"{self.config.bigquery_temp_table_schema}.ge-temp-{uuid.uuid4()}" - ) + num_parts = self.config.bigquery_temp_table_schema.split(".") + # If we only have 1 part that means the project_id is missing from the table name and we add it + if len(num_parts) == 1: + bigquery_temp_table = f"{temp_table_db}.{self.config.bigquery_temp_table_schema}.ge-temp-{uuid.uuid4()}" + elif len(num_parts) == 2: + bigquery_temp_table = f"{self.config.bigquery_temp_table_schema}.ge-temp-{uuid.uuid4()}" + else: + raise ConfigurationError( + f"bigquery_temp_table_schema should be either project.dataset or dataset format but it was: {self.config.bigquery_temp_table_schema}" + ) else: assert table table_parts = table.split(".") if len(table_parts) == 2: bigquery_temp_table = ( - f"{schema}.{table_parts[0]}.ge-temp-{uuid.uuid4()}" + f"{temp_table_db}.{table_parts[0]}.ge-temp-{uuid.uuid4()}" ) # With this pr there is no option anymore to set the bigquery temp table: @@ -941,6 +961,7 @@ def _get_ge_dataset( # }, # ) + logger.debug(f"Got pretty_name={pretty_name}, kwargs={batch_kwargs}") expectation_suite_name = ge_context.datasource_name + "." + pretty_name ge_context.data_context.create_expectation_suite( @@ -955,4 +976,18 @@ def _get_ge_dataset( **batch_kwargs, }, ) + if platform is not None and platform == "bigquery": + # This is done as GE makes the name as DATASET.TABLE + # but we want it to be PROJECT.DATASET.TABLE instead for multi-project setups + name_parts = pretty_name.split(".") + if len(name_parts) != 3: + logger.error( + f"Unexpected {pretty_name} while profiling. Should have 3 parts but has {len(name_parts)} parts." + ) + # If we only have two parts that means the project_id is missing from the table name and we add it + # Temp tables has 3 parts while normal tables only has 2 parts + if len(str(batch._table).split(".")) == 2: + batch._table = sa.text(f"{name_parts[0]}.{str(batch._table)}") + logger.debug(f"Setting table name to be {batch._table}") + return batch diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 28a8e04413932e..c58ed771c72887 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -84,6 +84,21 @@ class GEProfilingConfig(ConfigModel): description="A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", ) + profile_if_updated_since_days: Optional[pydantic.PositiveFloat] = Field( + default=1, + description="Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `Snowflake` and `BigQuery`.", + ) + + profile_table_size_limit: Optional[int] = Field( + default=1, + description="Profile tables only if their size is less then specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `BigQuery`", + ) + + profile_table_row_limit: Optional[int] = Field( + default=50000, + description="Profile tables only if their row count is less then specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `BigQuery`", + ) + # The default of (5 * cpu_count) is adopted from the default max_workers # parameter of ThreadPoolExecutor. Given that profiling is often an I/O-bound # task, it may make sense to increase this default value in the future. @@ -106,7 +121,7 @@ class GEProfilingConfig(ConfigModel): partition_profiling_enabled: bool = Field(default=True, description="") bigquery_temp_table_schema: Optional[str] = Field( default=None, - description="On bigquery for profiling partitioned tables needs to create temporary views. You have to define a schema where these will be created. Views will be cleaned up after profiler runs. (Great expectation tech details about this (https://legacy.docs.greatexpectations.io/en/0.9.0/reference/integrations/bigquery.html#custom-queries-with-sql-datasource).", + description="On bigquery for profiling partitioned tables needs to create temporary views. You have to define a dataset where these will be created. Views will be cleaned up after profiler runs. (Great expectation tech details about this (https://legacy.docs.greatexpectations.io/en/0.9.0/reference/integrations/bigquery.html#custom-queries-with-sql-datasource).", ) partition_datetime: Optional[datetime.datetime] = Field( default=None, diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py index d3bcc0222edf8d..ee2d7358909b4f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py @@ -313,16 +313,30 @@ def _parse_datatype(type: IcebergTypes.Type, nullable: bool = False) -> Dict[str "_nullable": nullable, } elif type.is_map_type(): + # The Iceberg Map type will be handled differently. The idea is to translate the map + # similar to the Map.Entry struct of Java i.e. as an array of map_entry struct, where + # the map_entry struct has a key field and a value field. The key and value type can + # be complex or primitive types. map_type: IcebergTypes.MapType = type - kt = _parse_datatype(map_type.key_type()) - vt = _parse_datatype(map_type.value_type()) - # keys are assumed to be strings in avro map + map_entry: Dict[str, Any] = { + "type": "record", + "name": _gen_name("__map_entry_"), + "fields": [ + { + "name": "key", + "type": _parse_datatype(map_type.key_type(), False), + }, + { + "name": "value", + "type": _parse_datatype(map_type.value_type(), True), + }, + ], + } return { - "type": "map", - "values": vt, - "native_data_type": str(map_type), - "key_type": kt, - "key_native_data_type": repr(map_type.key_type()), + "type": "array", + "items": map_entry, + "native_data_type": str(type), + "_nullable": nullable, } elif type.is_struct_type(): structType: IcebergTypes.StructType = type @@ -340,7 +354,7 @@ def _parse_struct_fields(parts: Tuple[NestedField], nullable: bool) -> Dict[str, fields.append({"name": field_name, "type": field_type, "doc": nested_field.doc}) return { "type": "record", - "name": "__struct_{}".format(str(uuid.uuid4()).replace("-", "")), + "name": _gen_name("__struct_"), "fields": fields, "native_data_type": "struct<{}>".format(parts), "_nullable": nullable, @@ -367,7 +381,7 @@ def _parse_basic_datatype( fixed_type: IcebergTypes.FixedType = type return { "type": "fixed", - "name": "name", # TODO: Pass-in field name since it is required by Avro spec + "name": _gen_name("__fixed_"), "size": fixed_type.length, "native_data_type": repr(fixed_type), "_nullable": nullable, @@ -380,7 +394,9 @@ def _parse_basic_datatype( return { # "type": "bytes", # when using bytes, avro drops _nullable attribute and others. See unit test. "type": "fixed", # to fix avro bug ^ resolved by using a fixed type - "name": "bogus", # to fix avro bug ^ resolved by using a fixed type + "name": _gen_name( + "__fixed_" + ), # to fix avro bug ^ resolved by using a fixed type "size": 1, # to fix avro bug ^ resolved by using a fixed type "logicalType": "decimal", "precision": decimal_type.precision, @@ -431,3 +447,7 @@ def _parse_basic_datatype( } return {"type": "null", "native_data_type": repr(type)} + + +def _gen_name(prefix: str) -> str: + return f"{prefix}{str(uuid.uuid4()).replace('-', '')}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py b/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py index 1fef58e2bafadf..202a2ba99390ca 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py +++ b/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py @@ -13,6 +13,7 @@ from datahub.configuration import ConfigModel from datahub.configuration.common import AllowDenyPattern from datahub.emitter.mce_builder import make_group_urn, make_user_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( # SourceCapability,; capability, SupportStatus, @@ -28,9 +29,12 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( + ChangeTypeClass, CorpGroupInfoClass, CorpUserInfoClass, GroupMembershipClass, + OriginClass, + OriginTypeClass, ) logger = logging.getLogger(__name__) @@ -288,6 +292,24 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.report.report_workunit(wu) yield wu + group_origin_mcp = MetadataChangeProposalWrapper( + entityType="corpGroup", + entityUrn=datahub_corp_group_snapshot.urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="origin", + aspect=OriginClass(OriginTypeClass.EXTERNAL, "AZURE_AD"), + ) + group_origin_wu_id = ( + f"group-origin-{group_count + 1}" + if self.config.mask_group_id + else datahub_corp_group_snapshot.urn + ) + group_origin_wu = MetadataWorkUnit( + id=group_origin_wu_id, mcp=group_origin_mcp + ) + self.report.report_workunit(group_origin_wu) + yield group_origin_wu + # Populate GroupMembership Aspects for CorpUsers datahub_corp_user_urn_to_group_membership: Dict[ str, GroupMembershipClass @@ -413,6 +435,22 @@ def ingest_ad_users( self.report.report_workunit(wu) yield wu + user_origin_mcp = MetadataChangeProposalWrapper( + entityType="corpuser", + entityUrn=datahub_corp_user_snapshot.urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="origin", + aspect=OriginClass(OriginTypeClass.EXTERNAL, "AZURE_AD"), + ) + user_origin_wu_id = ( + f"user-origin-{user_count + 1}" + if self.config.mask_user_id + else datahub_corp_user_snapshot.urn + ) + user_origin_wu = MetadataWorkUnit(id=user_origin_wu_id, mcp=user_origin_mcp) + self.report.report_workunit(user_origin_wu) + yield user_origin_wu + def get_report(self) -> SourceReport: return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py b/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py index af1ed371cab3d8..7923af70a8b8fd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py +++ b/metadata-ingestion/src/datahub/ingestion/source/identity/okta.py @@ -14,6 +14,7 @@ from datahub.configuration import ConfigModel from datahub.configuration.common import ConfigurationError +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -31,9 +32,12 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( # GroupMembershipClass, + ChangeTypeClass, CorpGroupInfoClass, CorpUserInfoClass, GroupMembershipClass, + OriginClass, + OriginTypeClass, ) logger = logging.getLogger(__name__) @@ -279,6 +283,24 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.report.report_workunit(wu) yield wu + group_origin_mcp = MetadataChangeProposalWrapper( + entityType="corpGroup", + entityUrn=datahub_corp_group_snapshot.urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="origin", + aspect=OriginClass(OriginTypeClass.EXTERNAL, "OKTA"), + ) + group_origin_wu_id = ( + f"group-origin-{group_count + 1}" + if self.config.mask_group_id + else datahub_corp_group_snapshot.urn + ) + group_origin_wu = MetadataWorkUnit( + id=group_origin_wu_id, mcp=group_origin_mcp + ) + self.report.report_workunit(group_origin_wu) + yield group_origin_wu + # Step 2: Populate GroupMembership Aspects for CorpUsers datahub_corp_user_urn_to_group_membership: Dict[str, GroupMembershipClass] = {} if self.config.ingest_group_membership and okta_groups is not None: @@ -352,6 +374,24 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.report.report_workunit(wu) yield wu + user_origin_mcp = MetadataChangeProposalWrapper( + entityType="corpuser", + entityUrn=datahub_corp_user_snapshot.urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="origin", + aspect=OriginClass(OriginTypeClass.EXTERNAL, "OKTA"), + ) + user_origin_wu_id = ( + f"user-origin-{user_count + 1}" + if self.config.mask_user_id + else datahub_corp_user_snapshot.urn + ) + user_origin_wu = MetadataWorkUnit( + id=user_origin_wu_id, mcp=user_origin_mcp + ) + self.report.report_workunit(user_origin_wu) + yield user_origin_wu + # Step 4: Close the event loop event_loop.close() diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka.py index 98551bd66e73cb..34e807a68ee567 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka.py @@ -1,7 +1,5 @@ import logging -import types from dataclasses import dataclass, field -from importlib import import_module from typing import Dict, Iterable, List, Optional, Type, cast import confluent_kafka @@ -26,6 +24,7 @@ platform_name, support_status, ) +from datahub.ingestion.api.registry import import_path from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase from datahub.ingestion.source.state.checkpoint import Checkpoint @@ -47,6 +46,7 @@ JobStatusClass, SubTypesClass, ) +from datahub.utilities.registries.domain_registry import DomainRegistry logger = logging.getLogger(__name__) @@ -119,11 +119,7 @@ def create_schema_registry( cls, config: KafkaSourceConfig, report: KafkaSourceReport ) -> KafkaSchemaRegistryBase: try: - module_path: str - class_name: str - module_path, class_name = config.schema_registry_class.rsplit(".", 1) - module: types.ModuleType = import_module(module_path) - schema_registry_class: Type = getattr(module, class_name) + schema_registry_class: Type = import_path(config.schema_registry_class) return schema_registry_class.create(config, report) except (ImportError, AttributeError): raise ImportError(config.schema_registry_class) @@ -150,6 +146,11 @@ def __init__(self, config: KafkaSourceConfig, ctx: PipelineContext): self.schema_registry_client: KafkaSchemaRegistryBase = ( KafkaSource.create_schema_registry(config, self.report) ) + if self.source_config.domain: + self.domain_registry = DomainRegistry( + cached_domains=[k for k in self.source_config.domain], + graph=self.ctx.graph, + ) def is_checkpointing_enabled(self, job_id: JobId) -> bool: if ( @@ -333,7 +334,9 @@ def _extract_record(self, topic: str) -> Iterable[MetadataWorkUnit]: # 6. Emit domains aspect MCPW for domain, pattern in self.source_config.domain.items(): if pattern.allowed(dataset_name): - domain_urn = make_domain_urn(domain) + domain_urn = make_domain_urn( + self.domain_registry.get_domain_urn(domain) + ) if domain_urn: wus = add_domain_to_entity_wu( diff --git a/metadata-ingestion/src/datahub/ingestion/source/ldap.py b/metadata-ingestion/src/datahub/ingestion/source/ldap.py index 633651c7b171a6..223b395f3aa965 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ldap.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ldap.py @@ -1,5 +1,6 @@ """LDAP Source""" import dataclasses +import re from typing import Any, Dict, Iterable, List, Optional import ldap @@ -22,8 +23,37 @@ CorpGroupSnapshotClass, CorpUserInfoClass, CorpUserSnapshotClass, + GroupMembershipClass, ) +# default mapping for attrs +user_attrs_map: Dict[str, Any] = {} +group_attrs_map: Dict[str, Any] = {} + +# general attrs +user_attrs_map["urn"] = "sAMAccountName" + +# user related attrs +user_attrs_map["fullName"] = "cn" +user_attrs_map["lastName"] = "sn" +user_attrs_map["firstName"] = "givenName" +user_attrs_map["displayName"] = "displayName" +user_attrs_map["managerUrn"] = "manager" +user_attrs_map["email"] = "mail" +user_attrs_map["departmentId"] = "departmentNumber" +user_attrs_map["title"] = "title" +user_attrs_map["departmentName"] = "departmentNumber" +user_attrs_map["countryCode"] = "countryCode" +user_attrs_map["memberOf"] = "memberOf" + +# group related attrs +group_attrs_map["urn"] = "cn" +group_attrs_map["email"] = "mail" +group_attrs_map["admins"] = "owner" +group_attrs_map["members"] = "uniqueMember" +group_attrs_map["displayName"] = "name" +group_attrs_map["description"] = "info" + def create_controls(pagesize: int) -> SimplePagedResultsControl: """ @@ -56,15 +86,6 @@ def set_cookie( return bool(cookie) -def guess_person_ldap(attrs: Dict[str, Any]) -> Optional[str]: - """Determine the user's LDAP based on the DN and attributes.""" - if "sAMAccountName" in attrs: - return attrs["sAMAccountName"][0].decode() - if "uid" in attrs: - return attrs["uid"][0].decode() - return None - - class LDAPSourceConfig(ConfigModel): """Config used by the LDAP Source.""" @@ -76,6 +97,7 @@ class LDAPSourceConfig(ConfigModel): # Extraction configuration. base_dn: str = Field(description="LDAP DN.") filter: str = Field(default="(objectClass=*)", description="LDAP extractor filter.") + attrs_list: List[str] = Field(default=None, description="Retrieved attributes list") # If set to true, any users without first and last names will be dropped. drop_missing_first_last_name: bool = Field( @@ -87,6 +109,10 @@ class LDAPSourceConfig(ConfigModel): default=20, description="Size of each page to fetch when extracting metadata." ) + # default mapping for attrs + user_attrs_map: Dict[str, Any] = {} + group_attrs_map: Dict[str, Any] = {} + @dataclasses.dataclass class LDAPSourceReport(SourceReport): @@ -97,6 +123,28 @@ def report_dropped(self, dn: str) -> None: self.dropped_dns.append(dn) +def guess_person_ldap( + attrs: Dict[str, Any], config: LDAPSourceConfig, report: LDAPSourceReport +) -> Optional[str]: + """Determine the user's LDAP based on the DN and attributes.""" + if config.user_attrs_map["urn"] in attrs: + return attrs[config.user_attrs_map["urn"]][0].decode() + else: # for backward compatiblity + if "sAMAccountName" in attrs: + report.report_warning( + "", + "Defaulting to sAMAccountName as it was found in attrs and not set in user_attrs_map in recipe", + ) + return attrs["sAMAccountName"][0].decode() + if "uid" in attrs: + report.report_warning( + "", + "Defaulting to uid as it was found in attrs and not set in user_attrs_map in recipe", + ) + return attrs["uid"][0].decode() + return None + + @platform_name("LDAP") @config_class(LDAPSourceConfig) @support_status(SupportStatus.CERTIFIED) @@ -116,6 +164,15 @@ def __init__(self, ctx: PipelineContext, config: LDAPSourceConfig): """Constructor.""" super().__init__(ctx) self.config = config + # ensure prior defaults are in place + for k in user_attrs_map: + if k not in self.config.user_attrs_map: + self.config.user_attrs_map[k] = user_attrs_map[k] + + for k in group_attrs_map: + if k not in self.config.group_attrs_map: + self.config.group_attrs_map[k] = group_attrs_map[k] + self.report = LDAPSourceReport() ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW) @@ -148,13 +205,12 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.config.base_dn, ldap.SCOPE_SUBTREE, self.config.filter, + self.config.attrs_list, serverctrls=[self.lc], ) _rtype, rdata, _rmsgid, serverctrls = self.ldap_client.result3(msgid) except ldap.LDAPError as e: - self.report.report_failure( - "ldap-control", "LDAP search failed: {}".format(e) - ) + self.report.report_failure("ldap-control", f"LDAP search failed: {e}") break for dn, attrs in rdata: @@ -177,6 +233,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: elif ( b"posixGroup" in attrs["objectClass"] or b"organizationalUnit" in attrs["objectClass"] + or b"groupOfNames" in attrs["objectClass"] or b"group" in attrs["objectClass"] ): yield from self.handle_group(dn, attrs) @@ -198,22 +255,21 @@ def handle_user(self, dn: str, attrs: Dict[str, Any]) -> Iterable[MetadataWorkUn work unit based on the information. """ manager_ldap = None - if "manager" in attrs: + if self.config.user_attrs_map["managerUrn"] in attrs: try: - m_cn = attrs["manager"][0].decode() + m_cn = attrs[self.config.user_attrs_map["managerUrn"]][0].decode() manager_msgid = self.ldap_client.search_ext( m_cn, ldap.SCOPE_BASE, self.config.filter, serverctrls=[self.lc], ) - _m_dn, m_attrs = self.ldap_client.result3(manager_msgid)[1][0] - manager_ldap = guess_person_ldap(m_attrs) + result = self.ldap_client.result3(manager_msgid) + if result[1]: + _m_dn, m_attrs = result[1][0] + manager_ldap = guess_person_ldap(m_attrs, self.config, self.report) except ldap.LDAPError as e: - self.report.report_warning( - dn, "manager LDAP search failed: {}".format(e) - ) - + self.report.report_warning(dn, f"manager LDAP search failed: {e}") mce = self.build_corp_user_mce(dn, attrs, manager_ldap) if mce: wu = MetadataWorkUnit(dn, mce) @@ -241,65 +297,107 @@ def build_corp_user_mce( """ Create the MetadataChangeEvent via DN and attributes. """ - ldap_user = guess_person_ldap(attrs) + ldap_user = guess_person_ldap(attrs, self.config, self.report) if self.config.drop_missing_first_last_name and ( - "givenName" not in attrs or "sn" not in attrs + self.config.user_attrs_map["firstName"] not in attrs + or self.config.user_attrs_map["lastName"] not in attrs ): return None - full_name = attrs["cn"][0].decode() - first_name = attrs["givenName"][0].decode() - last_name = attrs["sn"][0].decode() - - email = (attrs["mail"][0]).decode() if "mail" in attrs else ldap_user + full_name = attrs[self.config.user_attrs_map["fullName"]][0].decode() + first_name = attrs[self.config.user_attrs_map["firstName"]][0].decode() + last_name = attrs[self.config.user_attrs_map["lastName"]][0].decode() + groups = parse_groups(attrs, self.config.user_attrs_map["memberOf"]) + + email = ( + (attrs[self.config.user_attrs_map["email"]][0]).decode() + if self.config.user_attrs_map["email"] in attrs + else ldap_user + ) display_name = ( - (attrs["displayName"][0]).decode() if "displayName" in attrs else full_name + (attrs[self.config.user_attrs_map["displayName"]][0]).decode() + if self.config.user_attrs_map["displayName"] in attrs + else full_name + ) + department_id = ( + int(attrs[self.config.user_attrs_map["departmentId"]][0].decode()) + if self.config.user_attrs_map["departmentId"] in attrs + else None ) - department = ( - (attrs["departmentNumber"][0]).decode() - if "departmentNumber" in attrs + department_name = ( + (attrs[self.config.user_attrs_map["departmentName"]][0]).decode() + if self.config.user_attrs_map["departmentName"] in attrs + else None + ) + country_code = ( + (attrs[self.config.user_attrs_map["countryCode"]][0]).decode() + if self.config.user_attrs_map["countryCode"] in attrs + else None + ) + title = ( + attrs[self.config.user_attrs_map["title"]][0].decode() + if self.config.user_attrs_map["title"] in attrs else None ) - title = attrs["title"][0].decode() if "title" in attrs else None manager_urn = f"urn:li:corpuser:{manager_ldap}" if manager_ldap else None - return MetadataChangeEvent( - proposedSnapshot=CorpUserSnapshotClass( - urn=f"urn:li:corpuser:{ldap_user}", - aspects=[ - CorpUserInfoClass( - active=True, - email=email, - fullName=full_name, - firstName=first_name, - lastName=last_name, - departmentName=department, - displayName=display_name, - title=title, - managerUrn=manager_urn, - ) - ], - ) + user_snapshot = CorpUserSnapshotClass( + urn=f"urn:li:corpuser:{ldap_user}", + aspects=[ + CorpUserInfoClass( + active=True, + email=email, + fullName=full_name, + firstName=first_name, + lastName=last_name, + departmentId=department_id, + departmentName=department_name, + displayName=display_name, + countryCode=country_code, + title=title, + managerUrn=manager_urn, + ) + ], ) + if groups: + user_snapshot.aspects.append(GroupMembershipClass(groups=groups)) + + return MetadataChangeEvent(proposedSnapshot=user_snapshot) + def build_corp_group_mce(self, attrs: dict) -> Optional[MetadataChangeEvent]: """Creates a MetadataChangeEvent for LDAP groups.""" - cn = attrs.get("cn") + cn = attrs.get(self.config.group_attrs_map["urn"]) if cn: full_name = cn[0].decode() - owners = parse_from_attrs(attrs, "owner") - members = parse_from_attrs(attrs, "uniqueMember") - email = attrs["mail"][0].decode() if "mail" in attrs else full_name - + admins = parse_from_attrs(attrs, self.config.group_attrs_map["admins"]) + members = parse_from_attrs(attrs, self.config.group_attrs_map["members"]) + email = ( + attrs[self.config.group_attrs_map["email"]][0].decode() + if self.config.group_attrs_map["email"] in attrs + else full_name + ) + description = ( + attrs[self.config.group_attrs_map["description"]][0].decode() + if self.config.group_attrs_map["description"] in attrs + else None + ) + displayName = ( + attrs[self.config.group_attrs_map["displayName"]][0].decode() + if self.config.group_attrs_map["displayName"] in attrs + else None + ) return MetadataChangeEvent( proposedSnapshot=CorpGroupSnapshotClass( urn=f"urn:li:corpGroup:{full_name}", aspects=[ CorpGroupInfoClass( email=email, - admins=owners, + admins=admins, members=members, groups=[], + description=description, + displayName=displayName, ) ], ) @@ -329,3 +427,19 @@ def strip_ldap_info(input_clean: bytes) -> str: """Converts a b'uid=username,ou=Groups,dc=internal,dc=machines' format to username""" return input_clean.decode().split(",")[0].lstrip("uid=") + + +def parse_groups(attrs: Dict[str, Any], filter_key: str) -> List[str]: + """Converts a list of LDAP groups to Datahub corpgroup strings""" + if filter_key in attrs: + return [ + f"urn:li:corpGroup:{strip_ldap_group_cn(ldap_group)}" + for ldap_group in attrs[filter_key] + ] + return [] + + +def strip_ldap_group_cn(input_clean: bytes) -> str: + """Converts a b'cn=group_name,ou=Groups,dc=internal,dc=machines' + format to group name""" + return re.sub("cn=", "", input_clean.decode().split(",")[0], flags=re.IGNORECASE) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker.py b/metadata-ingestion/src/datahub/ingestion/source/looker.py index 812a4995f0bfbc..f85bfd865fb244 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker.py @@ -51,7 +51,11 @@ LookerExplore, LookerUtil, ) -from datahub.metadata.com.linkedin.pegasus2avro.common import ChangeAuditStamps, Status +from datahub.metadata.com.linkedin.pegasus2avro.common import ( + AuditStamp, + ChangeAuditStamps, + Status, +) from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import ( ChartSnapshot, DashboardSnapshot, @@ -59,16 +63,48 @@ from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( BrowsePathsClass, + CalendarIntervalClass, + ChangeTypeClass, ChartInfoClass, ChartTypeClass, DashboardInfoClass, + DashboardUsageStatisticsClass, + DashboardUserUsageCountsClass, OwnerClass, OwnershipClass, OwnershipTypeClass, + TimeWindowSizeClass, ) logger = logging.getLogger(__name__) +usage_queries: Dict[str, Dict] = { + # Query - group by dashboard and date, find unique users, dashboard runs count + "counts_per_day_per_dashboard": { + "model": "system__activity", + "view": "history", + "fields": [ + "history.dashboard_id", + "history.created_date", + "history.dashboard_user", + "history.dashboard_run_count", + ], + "filters": {}, + }, + # Query - group by user, dashboard and date, find runs count + "counts_per_day_per_user_per_dashboard": { + "model": "system__activity", + "view": "history", + "fields": [ + "history.created_date", + "history.dashboard_id", + "history.dashboard_run_count", + "user.id", + ], + "filters": {}, + }, +} + class TransportOptionsConfig(ConfigModel): timeout: int @@ -154,12 +190,21 @@ class LookerDashboardSourceConfig(LookerAPIConfig, LookerCommonConfig): None, description="Optional URL to use when constructing external URLs to Looker if the `base_url` is not the correct one to use. For example, `https://looker-public.company.com`. If not provided, the external base URL will default to `base_url`.", ) + extract_usage_history: bool = Field( + False, + description="Experimental (Subject to breaking change) -- Whether to ingest usage statistics for dashboards. Setting this to True will query looker system activity explores to fetch historical dashboard usage.", + ) + # TODO - stateful ingestion to autodetect usage history interval + extract_usage_history_for_interval: str = Field( + "1 day ago", + description="Experimental (Subject to breaking change) -- Used only if extract_usage_history is set to True. Interval to extract looker dashboard usage history for . https://docs.looker.com/reference/filter-expressions#date_and_time", + ) @validator("external_base_url", pre=True, always=True) def external_url_defaults_to_api_config_base_url( cls, v: Optional[str], *, values: Dict[str, Any], **kwargs: Dict[str, Any] - ) -> str: - return v or values["base_url"] + ) -> Optional[str]: + return v or values.get("base_url") @validator("platform_instance") def platform_instance_not_supported(cls, v: str) -> str: @@ -216,11 +261,11 @@ def url(self, base_url: str) -> str: # If the base_url contains a port number (like https://company.looker.com:19999) remove the port number m = re.match("^(.*):([0-9]+)$", base_url) if m is not None: - base_url = m.group(1) + base_url = m[1] if self.look_id is not None: - return base_url + "/looks/" + self.look_id + return f"{base_url}/looks/{self.look_id}" else: - return base_url + "/x/" + self.query_slug + return f"{base_url}/x/{self.query_slug}" def get_urn_element_id(self): # A dashboard element can use a look or just a raw query against an explore @@ -270,23 +315,22 @@ def __init__(self, client: Looker31SDK): def get_by_id( self, id: int, transport_options: Optional[TransportOptions] ) -> Optional[LookerUser]: - logger.debug("Will get user {}".format(id)) + logger.debug(f"Will get user {id}") if id in self.user_map: return self.user_map[id] - else: - try: - raw_user: User = self.client.user( - id, - fields=self.fields, - transport_options=transport_options, - ) - looker_user = LookerUser._from_user(raw_user) - self.user_map[id] = looker_user - return looker_user - except SDKError as e: - logger.warn("Could not find user with id {}".format(id)) - logger.warn("Failure was {}".format(e)) - return None + try: + raw_user: User = self.client.user( + id, + fields=self.fields, + transport_options=transport_options, + ) + looker_user = LookerUser._from_user(raw_user) + self.user_map[id] = looker_user + return looker_user + except SDKError as e: + logger.warning(f"Could not find user with id {id}") + logger.warning(f"Failure was {e}") + return None @dataclass @@ -301,13 +345,20 @@ class LookerDashboard: is_hidden: bool = False owner: Optional[LookerUser] = None strip_user_ids_from_email: Optional[bool] = True + last_updated_at: Optional[datetime.datetime] = None + last_updated_by: Optional[LookerUser] = None + deleted_at: Optional[datetime.datetime] = None + deleted_by: Optional[LookerUser] = None + favorite_count: Optional[int] = None + view_count: Optional[int] = None + last_viewed_at: Optional[datetime.datetime] = None def url(self, base_url): # If the base_url contains a port number (like https://company.looker.com:19999) remove the port number m = re.match("^(.*):([0-9]+)$", base_url) if m is not None: - base_url = m.group(1) - return base_url + "/dashboards/" + self.id + base_url = m[1] + return f"{base_url}/dashboards/{self.id}" def get_urn_dashboard_id(self): return f"dashboards.{self.id}" @@ -350,8 +401,7 @@ def _extract_view_from_field(field: str) -> str: assert ( field.count(".") == 1 ), f"Error: A field must be prefixed by a view name, field is: {field}" - view_name = field.split(".")[0] - return view_name + return field.split(".")[0] def _get_views_from_fields(self, fields: List[str]) -> List[str]: field_set = set(fields) @@ -441,7 +491,7 @@ def _get_fields_from_query(self, query: Optional[Query]) -> List[str]: def _get_looker_dashboard_element( # noqa: C901 self, element: DashboardElement ) -> Optional[LookerDashboardElement]: - # Dashboard elements can use raw queries against explores + # Dashboard elements can use raw usage_queries against explores explores: List[str] fields: List[str] @@ -449,17 +499,15 @@ def _get_looker_dashboard_element( # noqa: C901 raise ValueError("Element ID can't be None") if element.query is not None: - explores = [] fields = self._get_fields_from_query(element.query) - if element.query.view is not None: - # Get the explore from the view directly - explores = [element.query.view] - + # Get the explore from the view directly + explores = [element.query.view] if element.query.view is not None else [] logger.debug( "Element {}: Explores added: {}".format(element.title, explores) ) for exp in explores: self.explore_set.add((element.query.model, exp)) + return LookerDashboardElement( id=element.id, title=element.title if element.title is not None else "", @@ -693,12 +741,14 @@ def fetch_one_explore( def _make_dashboard_and_chart_mces( self, looker_dashboard: LookerDashboard - ) -> List[MetadataChangeEvent]: + ) -> Iterable[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]]: chart_mces = [ self._make_chart_mce(element, looker_dashboard) for element in looker_dashboard.dashboard_elements if element.type == "vis" ] + for chart_mce in chart_mces: + yield chart_mce dashboard_urn = builder.make_dashboard_urn( self.source_config.platform_name, looker_dashboard.get_urn_dashboard_id() @@ -712,7 +762,7 @@ def _make_dashboard_and_chart_mces( description=looker_dashboard.description or "", title=looker_dashboard.title, charts=[mce.proposedSnapshot.urn for mce in chart_mces], - lastModified=ChangeAuditStamps(), + lastModified=self._get_change_audit_stamps(looker_dashboard), dashboardUrl=looker_dashboard.url(self.source_config.external_base_url), ) @@ -730,8 +780,27 @@ def _make_dashboard_and_chart_mces( dashboard_snapshot.aspects.append(Status(removed=looker_dashboard.is_deleted)) dashboard_mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot) - - return chart_mces + [dashboard_mce] + yield dashboard_mce + + if self.source_config.extract_usage_history: + # Emit snapshot values of dashboard usage - do this always ? + dashboard_usage_mcp = MetadataChangeProposalWrapper( + entityType="dashboard", + entityUrn=dashboard_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="dashboardUsageStatistics", + aspect=DashboardUsageStatisticsClass( + timestampMillis=round(datetime.datetime.now().timestamp() * 1000), + favoritesCount=looker_dashboard.favorite_count, + viewsCount=looker_dashboard.view_count, + lastViewedAt=round( + looker_dashboard.last_viewed_at.timestamp() * 1000 + ) + if looker_dashboard.last_viewed_at + else None, + ), + ) + yield dashboard_usage_mcp def get_ownership( self, looker_dashboard: LookerDashboard @@ -752,6 +821,46 @@ def get_ownership( return ownership return None + def _get_change_audit_stamps( + self, looker_dashboard: LookerDashboard + ) -> ChangeAuditStamps: + change_audit_stamp: ChangeAuditStamps = ChangeAuditStamps() + if looker_dashboard.created_at is not None: + change_audit_stamp.created.time = round( + looker_dashboard.created_at.timestamp() * 1000 + ) + if looker_dashboard.owner is not None: + owner_urn = looker_dashboard.owner._get_urn( + self.source_config.strip_user_ids_from_email + ) + if owner_urn: + change_audit_stamp.created.actor = owner_urn + if looker_dashboard.last_updated_at is not None: + change_audit_stamp.lastModified.time = round( + looker_dashboard.last_updated_at.timestamp() * 1000 + ) + if looker_dashboard.last_updated_by is not None: + updated_by_urn = looker_dashboard.last_updated_by._get_urn( + self.source_config.strip_user_ids_from_email + ) + if updated_by_urn: + change_audit_stamp.lastModified.actor = updated_by_urn + if ( + looker_dashboard.is_deleted + and looker_dashboard.deleted_by is not None + and looker_dashboard.deleted_at is not None + ): + deleter_urn = looker_dashboard.deleted_by._get_urn( + self.source_config.strip_user_ids_from_email + ) + if deleter_urn: + change_audit_stamp.deleted = AuditStamp( + actor=deleter_urn, + time=round(looker_dashboard.deleted_at.timestamp() * 1000), + ) + + return change_audit_stamp + folder_path_cache: Dict[str, str] = {} def _get_folder_path(self, folder: FolderBase, client: Looker31SDK) -> str: @@ -800,36 +909,46 @@ def _get_looker_dashboard( if dashboard.id is None or dashboard.title is None: raise ValueError("Both dashboard ID and title are None") - dashboard_owner = ( + looker_dashboard = LookerDashboard( + id=dashboard.id, + title=dashboard.title, + description=dashboard.description, + dashboard_elements=dashboard_elements, + created_at=dashboard.created_at, + is_deleted=dashboard.deleted if dashboard.deleted is not None else False, + is_hidden=dashboard.hidden if dashboard.hidden is not None else False, + folder_path=dashboard_folder_path, + owner=self._get_looker_user(dashboard.user_id), + strip_user_ids_from_email=self.source_config.strip_user_ids_from_email, + last_updated_at=dashboard.updated_at, + last_updated_by=self._get_looker_user(dashboard.last_updater_id), + deleted_at=dashboard.deleted_at, + deleted_by=self._get_looker_user(dashboard.deleter_id), + favorite_count=dashboard.favorite_count, + view_count=dashboard.view_count, + last_viewed_at=dashboard.last_viewed_at, + ) + return looker_dashboard + + def _get_looker_user(self, user_id: Optional[int]) -> Optional[LookerUser]: + user = ( self.user_registry.get_by_id( - dashboard.user_id, + user_id, self.source_config.transport_options.get_transport_options() if self.source_config.transport_options is not None else None, ) - if self.source_config.extract_owners and dashboard.user_id is not None + if self.source_config.extract_owners and user_id is not None else None ) - if dashboard_owner is not None and self.source_config.extract_owners: + if user is not None and self.source_config.extract_owners: # Keep track of how many user ids we were able to resolve self.resolved_user_ids += 1 - if dashboard_owner.email is None: + if user.email is None: self.email_ids_missing += 1 - looker_dashboard = LookerDashboard( - id=dashboard.id, - title=dashboard.title, - description=dashboard.description, - dashboard_elements=dashboard_elements, - created_at=dashboard.created_at, - is_deleted=dashboard.deleted if dashboard.deleted is not None else False, - is_hidden=dashboard.deleted if dashboard.deleted is not None else False, - folder_path=dashboard_folder_path, - owner=dashboard_owner, - strip_user_ids_from_email=self.source_config.strip_user_ids_from_email, - ) - return looker_dashboard + return user def process_dashboard( self, dashboard_id: str @@ -847,11 +966,23 @@ def process_dashboard( "dashboard_elements", "dashboard_filters", "deleted", + "hidden", "description", "folder", "user_id", + "created_at", + "updated_at", + "last_updater_id", + "deleted_at", + "deleter_id", ] - dashboard_object = self.client.dashboard( + if self.source_config.extract_usage_history: + fields += [ + "favorite_count", + "view_count", + "last_viewed_at", + ] + dashboard_object: Dashboard = self.client.dashboard( dashboard_id=dashboard_id, fields=",".join(fields), transport_options=self.source_config.transport_options.get_transport_options() @@ -882,10 +1013,131 @@ def process_dashboard( # for mce in mces: workunits = [ MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce) + if isinstance(mce, MetadataChangeEvent) + else MetadataWorkUnit( + id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce + ) for mce in mces ] return workunits, dashboard_id, start_time, datetime.datetime.now() + def extract_usage_history_from_system_activity( + self, dashboard_ids: List[str] + ) -> Iterable[MetadataChangeProposalWrapper]: + + dashboard_ids_allowed = [ + dashboard_id + for dashboard_id in dashboard_ids + if self.source_config.dashboard_pattern.allowed(dashboard_id) + ] + + # key tuple (dashboard_id, date) + dashboard_usages: Dict[tuple, DashboardUsageStatisticsClass] = dict() + + common_filters = { + "history.dashboard_id": ",".join(dashboard_ids_allowed), + "history.created_date": self.source_config.extract_usage_history_for_interval, + } + for query in usage_queries.values(): + query["filters"].update(common_filters) + + self._populate_dashboard_counts(dashboard_usages) + + self._populate_userwise_runs_counts(dashboard_usages) + + for key, val in dashboard_usages.items(): + yield MetadataChangeProposalWrapper( + entityType="dashboard", + entityUrn=builder.make_dashboard_urn( + self.source_config.platform_name, + f"dashboards.{key[0]}", # in sync with LookerDashboard.get_urn_dashboard_id + ), + changeType=ChangeTypeClass.UPSERT, + aspectName="dashboardUsageStatistics", + aspect=val, + ) + + def _populate_userwise_runs_counts(self, dashboard_usages): + userwise_count_rows = LookerUtil.run_inline_query( + self.client, + usage_queries["counts_per_day_per_user_per_dashboard"], + transport_options=self.source_config.transport_options.get_transport_options() + if self.source_config.transport_options is not None + else None, + ) + + for row in userwise_count_rows: + user: Optional[LookerUser] = ( + self.user_registry.get_by_id( + row["user.id"], + self.source_config.transport_options.get_transport_options() + if self.source_config.transport_options is not None + else None, + ) + if row["user.id"] is not None + else None + ) + if user is None: + logger.warning( + f"Unable to resolve user with id {row['user.id']}, skipping" + ) + continue + + user_urn: Optional[str] = user._get_urn( + self.source_config.strip_user_ids_from_email + ) + + if user_urn is None: + logger.warning( + f"Unable to resolve urn for user with id {row['user.id']}, skipping" + ) + continue + + user_usage: DashboardUserUsageCountsClass = DashboardUserUsageCountsClass( + user=user_urn, + executionsCount=row["history.dashboard_run_count"], + usageCount=row["history.dashboard_run_count"], + userEmail=user.email, + ) + + usage_mcp_prev = dashboard_usages.get( + (row["history.dashboard_id"], row["history.created_date"]) + ) + if usage_mcp_prev is None: + # Unreachable + logger.warning( + f"User counts found but no users for {row['history.dashboard_id']} on date {row['history.created_date']}" + ) + continue + + if usage_mcp_prev.userCounts is None: + usage_mcp_prev.userCounts = [user_usage] + else: + usage_mcp_prev.userCounts.append(user_usage) + + def _populate_dashboard_counts(self, dashboard_usages): + count_rows = LookerUtil.run_inline_query( + self.client, + usage_queries["counts_per_day_per_dashboard"], + transport_options=self.source_config.transport_options.get_transport_options() + if self.source_config.transport_options is not None + else None, + ) + for row in count_rows: + dashboard_usages[ + (row["history.dashboard_id"], row["history.created_date"]) + ] = DashboardUsageStatisticsClass( + timestampMillis=round( + datetime.datetime.strptime(row["history.created_date"], "%Y-%m-%d") + .replace(tzinfo=datetime.timezone.utc) + .timestamp() + * 1000 + ), + eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY), + uniqueUserCount=row["history.dashboard_user"], + executionsCount=row["history.dashboard_run_count"], + ) + def get_workunits(self) -> Iterable[MetadataWorkUnit]: dashboards = self.client.all_dashboards( fields="id", @@ -957,7 +1209,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: else False, ) else: - raise Exception("Unexpected type of event {}".format(event)) + raise Exception(f"Unexpected type of event {event}") self.reporter.report_workunit(workunit) yield workunit @@ -972,5 +1224,17 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.reporter.report_workunit(workunit) yield workunit + if self.source_config.extract_usage_history and dashboard_ids is not None: + usage_mcps = self.extract_usage_history_from_system_activity( + dashboard_ids # type:ignore + ) + for usage_mcp in usage_mcps: + workunit = MetadataWorkUnit( + id=f"looker-{usage_mcp.aspectName}-{usage_mcp.entityUrn}-{usage_mcp.aspect.timestampMillis}", # type:ignore + mcp=usage_mcp, + ) + self.reporter.report_workunit(workunit) + yield workunit + def get_report(self) -> SourceReport: return self.reporter diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker_common.py index 5d20f6c96cb023..34b64242221f1b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker_common.py @@ -1,3 +1,4 @@ +import json import logging import re from dataclasses import dataclass @@ -8,6 +9,7 @@ from looker_sdk.error import SDKError from looker_sdk.rtl.transport import TransportOptions from looker_sdk.sdk.api31.methods import Looker31SDK +from looker_sdk.sdk.api31.models import WriteQuery from pydantic import BaseModel, Field from pydantic.class_validators import validator @@ -110,12 +112,8 @@ class LookerExploreNamingConfig(ConfigModel): def init_naming_pattern(cls, v): if isinstance(v, NamingPattern): return v - else: - assert isinstance(v, str), "pattern must be a string" - naming_pattern = NamingPattern( - allowed_vars=naming_pattern_variables, pattern=v - ) - return naming_pattern + assert isinstance(v, str), "pattern must be a string" + return NamingPattern(allowed_vars=naming_pattern_variables, pattern=v) @validator("explore_naming_pattern", "explore_browse_pattern", always=True) def validate_naming_pattern(cls, v): @@ -143,12 +141,8 @@ class LookerViewNamingConfig(ConfigModel): def init_naming_pattern(cls, v): if isinstance(v, NamingPattern): return v - else: - assert isinstance(v, str), "pattern must be a string" - naming_pattern = NamingPattern( - allowed_vars=naming_pattern_variables, pattern=v - ) - return naming_pattern + assert isinstance(v, str), "pattern must be a string" + return NamingPattern(allowed_vars=naming_pattern_variables, pattern=v) @validator("view_naming_pattern", "view_browse_pattern", always=True) def validate_naming_pattern(cls, v): @@ -314,8 +308,7 @@ def _extract_view_from_field(field: str) -> str: assert ( field.count(".") == 1 ), f"Error: A field must be prefixed by a view name, field is: {field}" - view_name = field.split(".")[0] - return view_name + return field.split(".")[0] @staticmethod def _get_field_type( @@ -336,8 +329,7 @@ def _get_field_type( ) type_class = NullTypeClass - data_type = SchemaFieldDataType(type=type_class()) - return data_type + return SchemaFieldDataType(type=type_class()) @staticmethod def _get_schema( @@ -346,7 +338,7 @@ def _get_schema( view_fields: List[ViewField], reporter: SourceReport, ) -> Optional[SchemaMetadataClass]: - if view_fields == []: + if not view_fields: return None fields, primary_keys = LookerUtil._get_fields_and_primary_keys( view_fields=view_fields, reporter=reporter @@ -467,6 +459,46 @@ def _display_name(name: str) -> str: """Returns a display name that corresponds to the Looker conventions""" return name.replace("_", " ").title() if name else name + @staticmethod + def create_query_request(q: dict, limit: Optional[str] = None) -> WriteQuery: + return WriteQuery( + model=q["model"], + view=q["view"], + fields=q.get("fields"), + filters=q.get("filters"), + filter_expression=q.get("filter_expressions"), + sorts=q.get("sorts"), + limit=q.get("limit") or limit, + column_limit=q.get("column_limit"), + vis_config={"type": "looker_column"}, + filter_config=q.get("filter_config"), + query_timezone="UTC", + ) + + @staticmethod + def run_inline_query( + client: Looker31SDK, q: dict, transport_options: Optional[TransportOptions] + ) -> List: + + response_sql = client.run_inline_query( + result_format="sql", + body=LookerUtil.create_query_request(q), + transport_options=transport_options, + ) + logger.debug("=================Query=================") + logger.debug(response_sql) + + response_json = client.run_inline_query( + result_format="json", + body=LookerUtil.create_query_request(q), + transport_options=transport_options, + ) + + logger.debug("=================Response=================") + data = json.loads(response_json) + logger.debug(f"length {len(data)}") + return data + @dataclass class LookerExplore: @@ -494,20 +526,23 @@ def _get_fields_from_sql_equality(sql_fragment: str) -> List[str]: return field_match.findall(sql_fragment) @classmethod - def __from_dict(cls, model_name: str, dict: Dict) -> "LookerExplore": + def from_dict(cls, model_name: str, dict: Dict) -> "LookerExplore": + view_names = set() + joins = None + # always add the explore's name or the name from the from clause as the view on which this explore is built + view_names.add(dict.get("from", dict.get("name"))) + if dict.get("joins", {}) != {}: + # additionally for join-based explores, pull in the linked views assert "joins" in dict - view_names = set() for join in dict["joins"]: + join_from = join.get("from") + view_names.add(join_from or join["name"]) sql_on = join.get("sql_on", None) if sql_on is not None: fields = cls._get_fields_from_sql_equality(sql_on) joins = fields - for f in fields: - view_names.add(LookerUtil._extract_view_from_field(f)) - else: - # non-join explore, get view_name from `from` field if possible, default to explore name - view_names = set(dict.get("from", dict.get("name"))) + return LookerExplore( model_name=model_name, name=dict["name"], @@ -543,7 +578,14 @@ def from_api( # noqa: C901 views.add(explore.name) if explore.joins is not None and explore.joins != []: - potential_views = [e.name for e in explore.joins if e.name is not None] + join_to_orig_name_map = {} + potential_views = [] + for e in explore.joins: + if e.from_ is not None: + potential_views.append(e.from_) + join_to_orig_name_map[e.name] = e.from_ + elif e.name is not None: + potential_views.append(e.name) for e_join in [ e for e in explore.joins if e.dependent_fields is not None ]: @@ -551,6 +593,9 @@ def from_api( # noqa: C901 for field_name in e_join.dependent_fields: try: view_name = LookerUtil._extract_view_from_field(field_name) + orig_name = join_to_orig_name_map.get(e_join.name) + if orig_name is not None: + view_name = orig_name potential_views.append(view_name) except AssertionError: reporter.report_warning( @@ -623,16 +668,13 @@ def from_api( # noqa: C901 source_file=explore.source_file, ) except SDKError as e: - logger.warn( - "Failed to extract explore {} from model {}.".format( - explore_name, model - ) + logger.warning( + f"Failed to extract explore {explore_name} from model {model}." ) logger.debug( - "Failed to extract explore {} from model {} with {}".format( - explore_name, model, e - ) + f"Failed to extract explore {explore_name} from model {model} with {e}" ) + except AssertionError: reporter.report_warning( key="chart-", @@ -683,7 +725,7 @@ def _get_url(self, base_url): # If the base_url contains a port number (like https://company.looker.com:19999) remove the port number m = re.match("^(.*):([0-9]+)$", base_url) if m is not None: - base_url = m.group(1) + base_url = m[1] return f"{base_url}/explore/{self.model_name}/{self.name}" def _to_metadata_events( # noqa: C901 diff --git a/metadata-ingestion/src/datahub/ingestion/source/lookml.py b/metadata-ingestion/src/datahub/ingestion/source/lookml.py index a08b0bf30bb100..df172c50316265 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/lookml.py +++ b/metadata-ingestion/src/datahub/ingestion/source/lookml.py @@ -1,5 +1,4 @@ import glob -import importlib import itertools import logging import pathlib @@ -25,8 +24,10 @@ platform_name, support_status, ) +from datahub.ingestion.api.registry import import_path from datahub.ingestion.source.looker_common import ( LookerCommonConfig, + LookerExplore, LookerUtil, LookerViewId, ViewField, @@ -130,18 +131,17 @@ def from_looker_connection( ".*": _get_generic_definition, } - if looker_connection.dialect_name is not None: - for extractor_pattern, extracting_function in extractors.items(): - if re.match(extractor_pattern, looker_connection.dialect_name): - (platform, db, schema) = extracting_function(looker_connection) - return cls(platform=platform, default_db=db, default_schema=schema) - raise ConfigurationError( - f"Could not find an appropriate platform for looker_connection: {looker_connection.name} with dialect: {looker_connection.dialect_name}" - ) - else: + if looker_connection.dialect_name is None: raise ConfigurationError( f"Unable to fetch a fully filled out connection for {looker_connection.name}. Please check your API permissions." ) + for extractor_pattern, extracting_function in extractors.items(): + if re.match(extractor_pattern, looker_connection.dialect_name): + (platform, db, schema) = extracting_function(looker_connection) + return cls(platform=platform, default_db=db, default_schema=schema) + raise ConfigurationError( + f"Could not find an appropriate platform for looker_connection: {looker_connection.name} with dialect: {looker_connection.dialect_name}" + ) class LookMLSourceConfig(LookerCommonConfig): @@ -177,6 +177,10 @@ class LookMLSourceConfig(LookerCommonConfig): 512000, # 512KB should be plenty description="When extracting the view definition from a lookml file, the maximum number of characters to extract.", ) + emit_reachable_views_only: bool = Field( + False, + description="When enabled, only views that are reachable from explores defined in the model files are emitted", + ) @validator("platform_instance") def platform_instance_not_supported(cls, v: str) -> str: @@ -341,7 +345,9 @@ def resolve_includes( or included_file.endswith(".dashboard.lookml") or included_file.endswith(".dashboard.lkml") ): - logger.debug(f"include '{inc}' is a dashboard, skipping it") + logger.debug( + f"include '{included_file}' is a dashboard, skipping it" + ) continue logger.debug( @@ -509,14 +515,10 @@ class LookerView: @classmethod def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]: assert "." in sql_parser_path, "sql_parser-path must contain a ." - module_name, cls_name = sql_parser_path.rsplit(".", 1) - import sys + parser_cls = import_path(sql_parser_path) - logger.debug(sys.path) - parser_cls = getattr(importlib.import_module(module_name), cls_name) if not issubclass(parser_cls, SQLParser): raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}") - return parser_cls @classmethod @@ -591,7 +593,7 @@ def from_looker_dict( if sql_table_name is not None else None ) - derived_table = looker_view.get("derived_table", None) + derived_table = looker_view.get("derived_table") dimensions = cls._get_fields( looker_view.get("dimensions", []), ViewFieldType.DIMENSION @@ -605,7 +607,7 @@ def from_looker_dict( fields: List[ViewField] = dimensions + dimension_groups + measures # also store the view logic and materialization - view_logic = looker_viewfile.raw_file_content[0:max_file_snippet_length] + view_logic = looker_viewfile.raw_file_content[:max_file_snippet_length] # Parse SQL from derived tables to extract dependencies if derived_table is not None: @@ -630,9 +632,7 @@ def from_looker_dict( if k in ["datagroup_trigger", "sql_trigger_value", "persist_for"]: materialized = True if "materialized_view" in derived_table: - materialized = ( - True if derived_table["materialized_view"] == "yes" else False - ) + materialized = derived_table["materialized_view"] == "yes" view_details = ViewProperties( materialized=materialized, viewLogic=view_logic, viewLanguage=view_lang @@ -653,15 +653,11 @@ def from_looker_dict( ) # If not a derived table, then this view essentially wraps an existing - # object in the database. - if sql_table_name is not None: - # If sql_table_name is set, there is a single dependency in the view, on the sql_table_name. - sql_table_names = [sql_table_name] - else: - # Otherwise, default to the view name as per the docs: - # https://docs.looker.com/reference/view-params/sql_table_name-for-view - sql_table_names = [view_name] - + # object in the database. If sql_table_name is set, there is a single + # dependency in the view, on the sql_table_name. + # Otherwise, default to the view name as per the docs: + # https://docs.looker.com/reference/view-params/sql_table_name-for-view + sql_table_names = [view_name] if sql_table_name is None else [sql_table_name] output_looker_view = LookerView( id=LookerViewId( project_name=project_name, model_name=model_name, view_name=view_name @@ -705,7 +701,7 @@ def _extract_metadata_from_sql_query( # Add those in if we detect that it is missing if not re.search(r"SELECT\s", sql_query, flags=re.I): # add a SELECT clause at the beginning - sql_query = "SELECT " + sql_query + sql_query = f"SELECT {sql_query}" if not re.search(r"FROM\s", sql_query, flags=re.I): # add a FROM clause at the end sql_query = f"{sql_query} FROM {sql_table_name if sql_table_name is not None else view_name}" @@ -714,7 +710,7 @@ def _extract_metadata_from_sql_query( sql_info = cls._get_sql_info(sql_query, sql_parser_path) sql_table_names = sql_info.table_names column_names = sql_info.column_names - if fields == []: + if not fields: # it seems like the view is defined purely as sql, let's try using the column names to populate the schema fields = [ # set types to unknown for now as our sql parser doesn't give us column types yet @@ -843,10 +839,7 @@ def _load_model(self, path: str) -> LookerModel: return looker_model def _platform_names_have_2_parts(self, platform: str) -> bool: - if platform in ["hive", "mysql", "athena"]: - return True - else: - return False + return platform in {"hive", "mysql", "athena"} def _generate_fully_qualified_name( self, sql_table_name: str, connection_def: LookerConnectionDefinition @@ -999,7 +992,6 @@ def _get_custom_properties(self, looker_view: LookerView) -> DatasetPropertiesCl def _build_dataset_mcps( self, looker_view: LookerView ) -> List[MetadataChangeProposalWrapper]: - events = [] subTypeEvent = MetadataChangeProposalWrapper( entityType="dataset", changeType=ChangeTypeClass.UPSERT, @@ -1007,7 +999,7 @@ def _build_dataset_mcps( aspectName="subTypes", aspect=SubTypesClass(typeNames=["view"]), ) - events.append(subTypeEvent) + events = [subTypeEvent] if looker_view.view_details is not None: viewEvent = MetadataChangeProposalWrapper( entityType="dataset", @@ -1048,9 +1040,7 @@ def _build_dataset_mce(self, looker_view: LookerView) -> MetadataChangeEvent: dataset_snapshot.aspects.append(schema_metadata) dataset_snapshot.aspects.append(self._get_custom_properties(looker_view)) - mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) - - return mce + return MetadataChangeEvent(proposedSnapshot=dataset_snapshot) def get_project_name(self, model_name: str) -> str: if self.source_config.project_name is not None: @@ -1081,9 +1071,10 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 str(self.source_config.base_folder), self.reporter ) - # some views can be mentioned by multiple 'include' statements, so this set is used to prevent - # creating duplicate MCE messages - processed_view_files: Set[str] = set() + # some views can be mentioned by multiple 'include' statements and can be included via different connections. + # So this set is used to prevent creating duplicate events + processed_view_map: Dict[str, Set[str]] = {} + view_connection_map: Dict[str, Tuple[str, str]] = {} # The ** means "this directory and all subdirectories", and hence should # include all the files we want. @@ -1092,7 +1083,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 for file_path in model_files: self.reporter.report_models_scanned() - model_name = file_path.stem[0:-model_suffix_len] + model_name = file_path.stem[:-model_suffix_len] if not self.source_config.model_pattern.allowed(model_name): self.reporter.report_models_dropped(model_name) @@ -1119,20 +1110,43 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 self.reporter.report_models_dropped(model_name) continue + explore_reachable_views = set() + for explore_dict in model.explores: + explore: LookerExplore = LookerExplore.from_dict( + model_name, explore_dict + ) + if explore.upstream_views: + for view_name in explore.upstream_views: + explore_reachable_views.add(view_name) + + processed_view_files = processed_view_map.get(model.connection) + if processed_view_files is None: + processed_view_map[model.connection] = set() + processed_view_files = processed_view_map[model.connection] + project_name = self.get_project_name(model_name) + logger.debug(f"Model: {model_name}; Includes: {model.resolved_includes}") for include in model.resolved_includes: logger.debug(f"Considering {include} for model {model_name}") if include in processed_view_files: logger.debug(f"view '{include}' already processed, skipping it") continue - logger.debug(f"Attempting to load view file: {include}") looker_viewfile = viewfile_loader.load_viewfile( include, connectionDefinition, self.reporter ) if looker_viewfile is not None: for raw_view in looker_viewfile.views: + if ( + self.source_config.emit_reachable_views_only + and raw_view["name"] not in explore_reachable_views + ): + logger.debug( + f"view {raw_view['name']} is not reachable from an explore, skipping.." + ) + continue + self.reporter.report_views_scanned() try: maybe_looker_view = LookerView.from_looker_dict( @@ -1157,24 +1171,49 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 if self.source_config.view_pattern.allowed( maybe_looker_view.id.view_name ): - mce = self._build_dataset_mce(maybe_looker_view) - workunit = MetadataWorkUnit( - id=f"lookml-view-{maybe_looker_view.id}", - mce=mce, + view_connection_mapping = view_connection_map.get( + maybe_looker_view.id.view_name ) - self.reporter.report_workunit(workunit) - processed_view_files.add(include) - yield workunit - - for mcp in self._build_dataset_mcps(maybe_looker_view): - # We want to treat mcp aspects as optional, so allowing failures in this aspect to be treated as warnings rather than failures + if not view_connection_mapping: + view_connection_map[ + maybe_looker_view.id.view_name + ] = (model_name, model.connection) + # first time we are discovering this view + mce = self._build_dataset_mce(maybe_looker_view) workunit = MetadataWorkUnit( - id=f"lookml-view-{mcp.aspectName}-{maybe_looker_view.id}", - mcp=mcp, - treat_errors_as_warnings=True, + id=f"lookml-view-{maybe_looker_view.id}", + mce=mce, ) + processed_view_files.add(include) self.reporter.report_workunit(workunit) yield workunit + for mcp in self._build_dataset_mcps( + maybe_looker_view + ): + # We want to treat mcp aspects as optional, so allowing failures in this aspect to be treated as warnings rather than failures + workunit = MetadataWorkUnit( + id=f"lookml-view-{mcp.aspectName}-{maybe_looker_view.id}", + mcp=mcp, + treat_errors_as_warnings=True, + ) + self.reporter.report_workunit(workunit) + yield workunit + else: + ( + prev_model_name, + prev_model_connection, + ) = view_connection_mapping + if prev_model_connection != model.connection: + # this view has previously been discovered and emitted using a different connection + logger.warning( + f"view {maybe_looker_view.id.view_name} from model {model_name}, connection {model.connection} was previously processed via model {prev_model_name}, connection {prev_model_connection} and will likely lead to incorrect lineage to the underlying tables" + ) + if ( + not self.source_config.emit_reachable_views_only + ): + logger.warning( + "Consider enabling the `emit_reachable_views_only` flag to handle this case." + ) else: self.reporter.report_views_dropped( str(maybe_looker_view.id) diff --git a/metadata-ingestion/src/datahub/ingestion/source/metabase.py b/metadata-ingestion/src/datahub/ingestion/source/metabase.py index 93308ff93b3226..98bcbcba591ebd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metabase.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metabase.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from functools import lru_cache from typing import Dict, Iterable, Optional @@ -199,7 +199,7 @@ def get_timestamp_millis_from_ts_string(ts_str: str) -> int: try: return int(dp.parse(ts_str).timestamp() * 1000) except (dp.ParserError, OverflowError): - return int(datetime.utcnow().timestamp() * 1000) + return int(datetime.now(timezone.utc).timestamp() * 1000) def construct_dashboard_from_api_data( self, dashboard_info: dict @@ -449,7 +449,7 @@ def get_datasource_urn(self, card_details): schema_name, table_name = self.get_source_table_from_id(source_table_id) if table_name: source_paths.add( - f"{schema_name + '.' if schema_name else ''}{table_name}" + f"{f'{schema_name}.' if schema_name else ''}{table_name}" ) else: try: @@ -478,7 +478,7 @@ def get_datasource_urn(self, card_details): # Create dataset URNs dataset_urn = [] - dbname = f"{database_name + '.' if database_name else ''}" + dbname = f"{f'{database_name}.' if database_name else ''}" source_tables = list(map(lambda tbl: f"{dbname}{tbl}", source_paths)) dataset_urn = [ builder.make_dataset_urn_with_platform_instance( diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py index 8d6201867dd8b3..9710fe023a2560 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py @@ -172,9 +172,9 @@ def construct_schema_pymongo( maximum size of the document that will be considered for generating the schema. """ - doc_size_field = "temporary_doc_size_field" aggregations: List[Dict] = [] if is_version_gte_4_4: + doc_size_field = "temporary_doc_size_field" # create a temporary field to store the size of the document. filter on it and then remove it. aggregations = [ {"$addFields": {doc_size_field: {"$bsonSize": "$$ROOT"}}}, diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py index bb8ac443555252..9677d5cbd3b5cb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py @@ -338,8 +338,8 @@ def __init__(self, config: NifiSourceConfig, ctx: PipelineContext) -> None: if self.config.site_url_to_site_name is None: self.config.site_url_to_site_name = {} if ( - not urljoin(self.config.site_url, "/nifi/") - in self.config.site_url_to_site_name + urljoin(self.config.site_url, "/nifi/") + not in self.config.site_url_to_site_name ): self.config.site_url_to_site_name[ urljoin(self.config.site_url, "/nifi/") @@ -620,7 +620,7 @@ def create_nifi_flow(self): if about_response.ok: nifi_version = about_response.json().get("about", {}).get("version") else: - logger.warn("Failed to fetch version for nifi") + logger.warning("Failed to fetch version for nifi") cluster_response = self.session.get( url=urljoin(self.config.site_url, CLUSTER_ENDPOINT) ) @@ -630,7 +630,7 @@ def create_nifi_flow(self): cluster_response.json().get("clusterSummary", {}).get("clustered") ) else: - logger.warn("Failed to fetch cluster summary for flow") + logger.warning("Failed to fetch cluster summary for flow") pg_response = self.session.get( url=urljoin(self.config.site_url, PG_ENDPOINT) + "root" ) @@ -715,7 +715,7 @@ def fetch_provenance_events( attempts = 5 # wait for at most 5 attempts 5*1= 5 seconds while (not provenance.get("finished", False)) and attempts > 0: - logger.warn( + logger.warning( f"Provenance query not completed, attempts left : {attempts}" ) # wait until the uri returns percentcomplete 100 @@ -757,7 +757,7 @@ def fetch_provenance_events( f"provenance events could not be fetched for processor \ {processor.id} of type {processor.name}", ) - logger.warn(provenance_response.text) + logger.warning(provenance_response.text) return def report_warning(self, key: str, reason: str) -> None: @@ -774,7 +774,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 rootpg = self.nifi_flow.root_process_group flow_name = rootpg.name # self.config.site_name flow_urn = builder.make_data_flow_urn(NIFI, rootpg.id, self.config.env) - flow_properties = dict() + flow_properties = {} if self.nifi_flow.clustered is not None: flow_properties["clustered"] = str(self.nifi_flow.clustered) if self.nifi_flow.version is not None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/openapi.py b/metadata-ingestion/src/datahub/ingestion/source/openapi.py index b71cb363b96e46..9548677e1cdc11 100755 --- a/metadata-ingestion/src/datahub/ingestion/source/openapi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/openapi.py @@ -118,7 +118,7 @@ class ApiWorkUnit(MetadataWorkUnit): class APISource(Source, ABC): """ - This plugin is meant to gather dataset-like informations about OpenApi Endpoints. + This plugin is meant to gather dataset-like information about OpenApi Endpoints. As example, if by calling GET at the endpoint at `https://test_endpoint.com/api/users/` you obtain as result: ```JSON diff --git a/metadata-ingestion/src/datahub/ingestion/source/openapi_parser.py b/metadata-ingestion/src/datahub/ingestion/source/openapi_parser.py index 830b6562755eb7..f33654daa15595 100755 --- a/metadata-ingestion/src/datahub/ingestion/source/openapi_parser.py +++ b/metadata-ingestion/src/datahub/ingestion/source/openapi_parser.py @@ -20,9 +20,9 @@ def flatten(d: dict, prefix: str = "") -> Generator: for k, v in d.items(): if isinstance(v, dict): - yield from flatten(v, prefix + "." + k) + yield from flatten(v, f"{prefix}.{k}") else: - yield (prefix + "-" + k).strip(".") + yield f"{prefix}-{k}".strip(".") def flatten2list(d: dict) -> list: @@ -53,15 +53,15 @@ def request_call( headers = {"accept": "application/json"} if username is not None and password is not None: - response = requests.get( + return requests.get( url, headers=headers, auth=HTTPBasicAuth(username, password) ) + elif token is not None: - headers["Authorization"] = "Bearer " + token - response = requests.get(url, headers=headers) + headers["Authorization"] = f"Bearer {token}" + return requests.get(url, headers=headers) else: - response = requests.get(url, headers=headers) - return response + return requests.get(url, headers=headers) def get_swag_json( @@ -77,14 +77,13 @@ def get_swag_json( else: response = request_call(url=tot_url, username=username, password=password) - if response.status_code == 200: - try: - dict_data = json.loads(response.content) - except json.JSONDecodeError: # it's not a JSON! - dict_data = yaml.safe_load(response.content) - return dict_data - else: + if response.status_code != 200: raise Exception(f"Unable to retrieve {tot_url}, error {response.status_code}") + try: + dict_data = json.loads(response.content) + except json.JSONDecodeError: # it's not a JSON! + dict_data = yaml.safe_load(response.content) + return dict_data def get_url_basepath(sw_dict: dict) -> str: @@ -95,7 +94,7 @@ def get_url_basepath(sw_dict: dict) -> str: def check_sw_version(sw_dict: dict) -> None: - if "swagger" in sw_dict.keys(): + if "swagger" in sw_dict: v_split = sw_dict["swagger"].split(".") else: v_split = sw_dict["openapi"].split(".") @@ -176,8 +175,7 @@ def get_endpoints(sw_dict: dict) -> dict: # noqa: C901 if "parameters" in p_o["get"].keys(): url_details[p_k]["parameters"] = p_o["get"]["parameters"] - ord_d = dict(sorted(url_details.items())) # sorting for convenience - return ord_d + return dict(sorted(url_details.items())) def guessing_url_name(url: str, examples: dict) -> str: @@ -187,10 +185,7 @@ def guessing_url_name(url: str, examples: dict) -> str: extr_data = {"advancedcomputersearches": {'id': 202, 'name': '_unmanaged'}} -->> guessed_url = /advancedcomputersearches/name/_unmanaged/id/202' """ - if url[0] == "/": - url2op = url[1:] # operational url does not need the very first / - else: - url2op = url + url2op = url[1:] if url[0] == "/" else url divisions = url2op.split("/") # the very first part of the url should stay the same. @@ -211,14 +206,14 @@ def guessing_url_name(url: str, examples: dict) -> str: if div_pos > 0: root = root[: div_pos - 1] # like "base/field" should become "base" - if root in examples.keys(): + if root in examples: # if our root is contained in our samples examples... ex2use = root - elif root[:-1] in examples.keys(): + elif root[:-1] in examples: ex2use = root[:-1] - elif root.replace("/", ".") in examples.keys(): + elif root.replace("/", ".") in examples: ex2use = root.replace("/", ".") - elif root[:-1].replace("/", ".") in examples.keys(): + elif root[:-1].replace("/", ".") in examples: ex2use = root[:-1].replace("/", ".") else: return url @@ -277,8 +272,7 @@ def try_guessing(url: str, examples: dict) -> str: Any non-guessed name will stay as it was (with parenthesis{}) """ url_guess = guessing_url_name(url, examples) # try to fill with known informations - url_guess_id = maybe_theres_simple_id(url_guess) # try to fill IDs with "1"s... - return url_guess_id + return maybe_theres_simple_id(url_guess) def clean_url(url: str) -> str: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py index 5cfba5fa2ec14b..9e92d0056d8a6b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py @@ -13,12 +13,12 @@ from xmlrpc.client import Boolean import msal +import pydantic import requests from orderedset import OrderedSet -from pydantic.fields import Field import datahub.emitter.mce_builder as builder -from datahub.configuration.common import AllowDenyPattern, ConfigurationError +from datahub.configuration.common import ConfigurationError from datahub.configuration.source_common import EnvBasedSourceConfigBase from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext @@ -32,10 +32,7 @@ ) from datahub.ingestion.api.source import Source, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit -from datahub.metadata.com.linkedin.pegasus2avro.common import ( - ChangeAuditStamps, - DataPlatformInstance, -) +from datahub.metadata.com.linkedin.pegasus2avro.common import ChangeAuditStamps from datahub.metadata.schema_classes import ( BrowsePathsClass, ChangeTypeClass, @@ -104,41 +101,39 @@ class Constant: VALUE = "value" ENTITY = "ENTITY" ID = "ID" + HTTP_RESPONSE_TEXT = "HttpResponseText" + HTTP_RESPONSE_STATUS_CODE = "HttpResponseStatusCode" class PowerBiAPIConfig(EnvBasedSourceConfigBase): # Organsation Identifier - tenant_id: str = Field(description="Power BI tenant identifier.") + tenant_id: str = pydantic.Field(description="PowerBI tenant identifier") # PowerBi workspace identifier - workspace_id: str = Field(description="Power BI workspace identifier.") + workspace_id: str = pydantic.Field(description="PowerBI workspace identifier") # Dataset type mapping - dataset_type_mapping: Dict[str, str] = Field( - description="Mapping of Power BI datasource type to Datahub dataset." + dataset_type_mapping: Dict[str, str] = pydantic.Field( + description="Mapping of PowerBI datasource type to DataHub supported data-sources. See Quickstart Recipe for mapping" ) # Azure app client identifier - client_id: str = Field(description="Azure AD App client identifier.") + client_id: str = pydantic.Field(description="Azure app client identifier") # Azure app client secret - client_secret: str = Field(description="Azure AD App client secret.") + client_secret: str = pydantic.Field(description="Azure app client secret") # timeout for meta-data scanning - scan_timeout: int = Field( - default=60, - description="time in seconds to wait for Power BI metadata scan result.", + scan_timeout: int = pydantic.Field( + default=60, description="timeout for PowerBI metadata scanning" + ) + # Enable/Disable extracting ownership information of Dashboard + extract_ownership: bool = pydantic.Field( + default=True, description="Whether ownership should be ingested" ) - - scope: str = "https://analysis.windows.net/powerbi/api/.default" - base_url: str = "https://api.powerbi.com/v1.0/myorg/groups" - admin_base_url = "https://api.powerbi.com/v1.0/myorg/admin" - authority = "https://login.microsoftonline.com/" - - def get_authority_url(self): - return "{}{}".format(self.authority, self.tenant_id) class PowerBiDashboardSourceConfig(PowerBiAPIConfig): platform_name: str = "powerbi" platform_urn: str = builder.make_data_platform_urn(platform=platform_name) - dashboard_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() - chart_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() + # Not supporting the pattern + # dashboard_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() + # chart_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() class PowerBiAPI: @@ -155,6 +150,11 @@ class PowerBiAPI: Constant.SCAN_CREATE: "{POWERBI_ADMIN_BASE_URL}/workspaces/getInfo", } + SCOPE: str = "https://analysis.windows.net/powerbi/api/.default" + BASE_URL: str = "https://api.powerbi.com/v1.0/myorg/groups" + ADMIN_BASE_URL: str = "https://api.powerbi.com/v1.0/myorg/admin" + AUTHORITY: str = "https://login.microsoftonline.com/" + @dataclass class Workspace: """ @@ -216,7 +216,7 @@ class Table: tables: List[Any] def get_urn_part(self): - return "datasets.{}".format(self.id) + return f"datasets.{self.id}" def __members(self): return (self.id,) @@ -239,7 +239,7 @@ class Report: dataset: Any def get_urn_part(self): - return "reports.{}".format(self.id) + return f"reports.{self.id}" @dataclass class Tile: @@ -257,7 +257,7 @@ class CreatedFrom(Enum): createdFrom: CreatedFrom def get_urn_part(self): - return "charts.{}".format(self.id) + return f"charts.{self.id}" @dataclass class User: @@ -269,7 +269,7 @@ class User: principalType: str def get_urn_part(self): - return "users.{}".format(self.id) + return f"users.{self.id}" def __members(self): return (self.id,) @@ -296,7 +296,7 @@ class Dashboard: users: List[Any] def get_urn_part(self): - return "dashboards.{}".format(self.id) + return f"dashboards.{self.id}" def __members(self): return (self.id,) @@ -318,48 +318,57 @@ def __init__(self, config: PowerBiAPIConfig) -> None: self.__msal_client = msal.ConfidentialClientApplication( self.__config.client_id, client_credential=self.__config.client_secret, - authority=self.__config.authority + self.__config.tenant_id, + authority=PowerBiAPI.AUTHORITY + self.__config.tenant_id, ) # Test connection by generating a access token - LOGGER.info("Trying to connect to {}".format(self.__config.get_authority_url())) + LOGGER.info("Trying to connect to {}".format(self.__get_authority_url())) self.get_access_token() - LOGGER.info("Able to connect to {}".format(self.__config.get_authority_url())) + LOGGER.info("Able to connect to {}".format(self.__get_authority_url())) + + def __get_authority_url(self): + return "{}{}".format(PowerBiAPI.AUTHORITY, self.__config.tenant_id) def __get_users(self, workspace_id: str, entity: str, id: str) -> List[User]: """ Get user for the given PowerBi entity """ + users: List[PowerBiAPI.User] = [] + if self.__config.extract_ownership is False: + LOGGER.info( + "ExtractOwnership capabilities is disabled from configuration and hence returning empty users list" + ) + return users + user_list_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.ENTITY_USER_LIST] # Replace place holders user_list_endpoint = user_list_endpoint.format( - POWERBI_ADMIN_BASE_URL=self.__config.admin_base_url, + POWERBI_ADMIN_BASE_URL=PowerBiAPI.ADMIN_BASE_URL, ENTITY=entity, ENTITY_ID=id, ) # Hit PowerBi - LOGGER.info("Request to URL={}".format(user_list_endpoint)) + LOGGER.info(f"Request to URL={user_list_endpoint}") response = requests.get( - url=user_list_endpoint, + user_list_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) # Check if we got response from PowerBi if response.status_code != 200: LOGGER.warning( - "Failed to fetch user list from power-bi for, http_status={}, message={}".format( - response.status_code, response.text - ) + f"Failed to fetch user list from power-bi for, http_status={response.status_code}, message={response.text}" ) - LOGGER.info("{}={}".format(Constant.WorkspaceId, workspace_id)) - LOGGER.info("{}={}".format(Constant.ENTITY, entity)) - LOGGER.info("{}={}".format(Constant.ID, id)) + + LOGGER.info(f"{Constant.WorkspaceId}={workspace_id}") + LOGGER.info(f"{Constant.ENTITY}={entity}") + LOGGER.info(f"{Constant.ID}={id}") raise ConnectionError("Failed to fetch the user list from the power-bi") users_dict: List[Any] = response.json()[Constant.VALUE] # Iterate through response and create a list of PowerBiAPI.Dashboard - users: List[PowerBiAPI.User] = [ + users = [ PowerBiAPI.User( id=instance.get("identifier"), displayName=instance.get("displayName"), @@ -379,21 +388,21 @@ def __get_report(self, workspace_id: str, report_id: str) -> Any: """ if workspace_id is None or report_id is None: LOGGER.info("Input values are None") - LOGGER.info("{}={}".format(Constant.WorkspaceId, workspace_id)) - LOGGER.info("{}={}".format(Constant.ReportId, report_id)) + LOGGER.info(f"{Constant.WorkspaceId}={workspace_id}") + LOGGER.info(f"{Constant.ReportId}={report_id}") return None report_get_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.REPORT_GET] # Replace place holders report_get_endpoint = report_get_endpoint.format( - POWERBI_BASE_URL=self.__config.base_url, + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, WORKSPACE_ID=workspace_id, REPORT_ID=report_id, ) # Hit PowerBi - LOGGER.info("Request to report URL={}".format(report_get_endpoint)) + LOGGER.info(f"Request to report URL={report_get_endpoint}") response = requests.get( - url=report_get_endpoint, + report_get_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) @@ -401,8 +410,8 @@ def __get_report(self, workspace_id: str, report_id: str) -> Any: if response.status_code != 200: message: str = "Failed to fetch report from power-bi for" LOGGER.warning(message) - LOGGER.warning("{}={}".format(Constant.WorkspaceId, workspace_id)) - LOGGER.warning("{}={}".format(Constant.ReportId, report_id)) + LOGGER.warning(f"{Constant.WorkspaceId}={workspace_id}") + LOGGER.warning(f"{Constant.ReportId}={report_id}") raise ConnectionError(message) response_dict = response.json() @@ -425,7 +434,7 @@ def get_access_token(self): LOGGER.info("Generating PowerBi access token") auth_response = self.__msal_client.acquire_token_for_client( - scopes=[self.__config.scope] + scopes=[PowerBiAPI.SCOPE] ) if not auth_response.get("access_token"): @@ -440,7 +449,7 @@ def get_access_token(self): self.__access_token = "Bearer {}".format(auth_response.get("access_token")) - LOGGER.debug("{}={}".format(Constant.PBIAccessToken, self.__access_token)) + LOGGER.debug(f"{Constant.PBIAccessToken}={self.__access_token}") return self.__access_token @@ -461,19 +470,19 @@ def get_dashboards(self, workspace: Workspace) -> List[Dashboard]: dashboard_list_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.DASHBOARD_LIST] # Replace place holders dashboard_list_endpoint = dashboard_list_endpoint.format( - POWERBI_BASE_URL=self.__config.base_url, WORKSPACE_ID=workspace.id + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, WORKSPACE_ID=workspace.id ) # Hit PowerBi - LOGGER.info("Request to URL={}".format(dashboard_list_endpoint)) + LOGGER.info(f"Request to URL={dashboard_list_endpoint}") response = requests.get( - url=dashboard_list_endpoint, + dashboard_list_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) # Check if we got response from PowerBi if response.status_code != 200: LOGGER.warning("Failed to fetch dashboard list from power-bi for") - LOGGER.warning("{}={}".format(Constant.WorkspaceId, workspace.id)) + LOGGER.warning(f"{Constant.WorkspaceId}={workspace.id}") raise ConnectionError( "Failed to fetch the dashboard list from the power-bi" ) @@ -505,21 +514,21 @@ def get_dataset(self, workspace_id: str, dataset_id: str) -> Any: """ if workspace_id is None or dataset_id is None: LOGGER.info("Input values are None") - LOGGER.info("{}={}".format(Constant.WorkspaceId, workspace_id)) - LOGGER.info("{}={}".format(Constant.DatasetId, dataset_id)) + LOGGER.info(f"{Constant.WorkspaceId}={workspace_id}") + LOGGER.info(f"{Constant.DatasetId}={dataset_id}") return None dataset_get_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.DATASET_GET] # Replace place holders dataset_get_endpoint = dataset_get_endpoint.format( - POWERBI_BASE_URL=self.__config.base_url, + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, WORKSPACE_ID=workspace_id, DATASET_ID=dataset_id, ) # Hit PowerBi - LOGGER.info("Request to dataset URL={}".format(dataset_get_endpoint)) + LOGGER.info(f"Request to dataset URL={dataset_get_endpoint}") response = requests.get( - url=dataset_get_endpoint, + dataset_get_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) @@ -527,12 +536,12 @@ def get_dataset(self, workspace_id: str, dataset_id: str) -> Any: if response.status_code != 200: message: str = "Failed to fetch dataset from power-bi for" LOGGER.warning(message) - LOGGER.warning("{}={}".format(Constant.WorkspaceId, workspace_id)) - LOGGER.warning("{}={}".format(Constant.DatasetId, dataset_id)) + LOGGER.warning(f"{Constant.WorkspaceId}={workspace_id}") + LOGGER.warning(f"{Constant.DatasetId}={dataset_id}") raise ConnectionError(message) response_dict = response.json() - + LOGGER.debug("datasets = {}".format(response_dict)) # PowerBi Always return the webURL, in-case if it is None then setting complete webURL to None instead of None/details return PowerBiAPI.Dataset( id=response_dict.get("id"), @@ -553,14 +562,14 @@ def get_data_source(self, dataset: Dataset) -> Any: datasource_get_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.DATASOURCE_GET] # Replace place holders datasource_get_endpoint = datasource_get_endpoint.format( - POWERBI_BASE_URL=self.__config.base_url, + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, WORKSPACE_ID=dataset.workspace_id, DATASET_ID=dataset.id, ) # Hit PowerBi - LOGGER.info("Request to datasource URL={}".format(datasource_get_endpoint)) + LOGGER.info(f"Request to datasource URL={datasource_get_endpoint}") response = requests.get( - url=datasource_get_endpoint, + datasource_get_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) @@ -570,23 +579,40 @@ def get_data_source(self, dataset: Dataset) -> Any: LOGGER.warning(message) LOGGER.warning("{}={}".format(Constant.WorkspaceId, dataset.workspace_id)) LOGGER.warning("{}={}".format(Constant.DatasetId, dataset.id)) + LOGGER.warning("{}={}".format(Constant.HTTP_RESPONSE_TEXT, response.text)) + LOGGER.warning( + "{}={}".format(Constant.HTTP_RESPONSE_STATUS_CODE, response.status_code) + ) + raise ConnectionError(message) res = response.json() value = res["value"] if len(value) == 0: LOGGER.info( - "datasource is not found for dataset {}({})".format( + f"datasource is not found for dataset {dataset.name}({dataset.id})" + ) + + return None + + if len(value) > 1: + # We are currently supporting data-set having single relational database + LOGGER.warning( + "More than one data-source found for {}({})".format( dataset.name, dataset.id ) ) + LOGGER.debug(value) return None + # Consider only zero index datasource datasource_dict = value[0] - + LOGGER.debug("data-sources = {}".format(value)) # Create datasource instance with basic detail available datasource = PowerBiAPI.DataSource( - id=datasource_dict["datasourceId"], + id=datasource_dict.get( + "datasourceId" + ), # datasourceId is not available in all cases type=datasource_dict["datasourceType"], server=None, database=None, @@ -601,6 +627,9 @@ def get_data_source(self, dataset: Dataset) -> Any: datasource.server = datasource_dict["connectionDetails"]["server"] else: datasource.metadata = PowerBiAPI.DataSource.MetaData(is_relational=False) + LOGGER.warning( + "Non relational data-source found = {}".format(datasource_dict) + ) return datasource @@ -617,27 +646,23 @@ def new_dataset_or_report(tile_instance: Any) -> dict: Find out which is the data source for tile. It is either REPORT or DATASET """ report_fields = { - "dataset": None, - "report": None, + "dataset": ( + workspace.datasets[tile_instance.get("datasetId")] + if tile_instance.get("datasetId") is not None + else None + ), + "report": ( + self.__get_report( + workspace_id=workspace.id, + report_id=tile_instance.get("reportId"), + ) + if tile_instance.get("reportId") is not None + else None + ), "createdFrom": PowerBiAPI.Tile.CreatedFrom.UNKNOWN, } - report_fields["dataset"] = ( - workspace.datasets[tile_instance.get("datasetId")] - if tile_instance.get("datasetId") is not None - else None - ) - report_fields["report"] = ( - self.__get_report( - workspace_id=workspace.id, - report_id=tile_instance.get("reportId"), - ) - if tile_instance.get("reportId") is not None - else None - ) - # Tile is either created from report or dataset or from custom visualization - report_fields["createdFrom"] = PowerBiAPI.Tile.CreatedFrom.UNKNOWN if report_fields["report"] is not None: report_fields["createdFrom"] = PowerBiAPI.Tile.CreatedFrom.REPORT elif report_fields["dataset"] is not None: @@ -646,11 +671,7 @@ def new_dataset_or_report(tile_instance: Any) -> dict: report_fields["createdFrom"] = PowerBiAPI.Tile.CreatedFrom.VISUALIZATION LOGGER.info( - "Tile {}({}) is created from {}".format( - tile_instance.get("title"), - tile_instance.get("id"), - report_fields["createdFrom"], - ) + f'Tile {tile_instance.get("title")}({tile_instance.get("id")}) is created from {report_fields["createdFrom"]}' ) return report_fields @@ -658,14 +679,14 @@ def new_dataset_or_report(tile_instance: Any) -> dict: tile_list_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.TILE_LIST] # Replace place holders tile_list_endpoint = tile_list_endpoint.format( - POWERBI_BASE_URL=self.__config.base_url, + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, WORKSPACE_ID=dashboard.workspace_id, DASHBOARD_ID=dashboard.id, ) # Hit PowerBi LOGGER.info("Request to URL={}".format(tile_list_endpoint)) response = requests.get( - url=tile_list_endpoint, + tile_list_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) @@ -678,6 +699,7 @@ def new_dataset_or_report(tile_instance: Any) -> dict: # Iterate through response and create a list of PowerBiAPI.Dashboard tile_dict: List[Any] = response.json()[Constant.VALUE] + LOGGER.debug("Tile Dict = {}".format(tile_dict)) tiles: List[PowerBiAPI.Tile] = [ PowerBiAPI.Tile( id=instance.get("id"), @@ -698,7 +720,7 @@ def get_workspace(self, workspace_id: str) -> Workspace: """ scan_create_endpoint = PowerBiAPI.API_ENDPOINTS[Constant.SCAN_CREATE] scan_create_endpoint = scan_create_endpoint.format( - POWERBI_ADMIN_BASE_URL=self.__config.admin_base_url + POWERBI_ADMIN_BASE_URL=PowerBiAPI.ADMIN_BASE_URL ) def create_scan_job(): @@ -721,9 +743,7 @@ def create_scan_job(): ) if res.status_code not in (200, 202): - message = "API({}) return error code {} for workpace id({})".format( - scan_create_endpoint, res.status_code, workspace_id - ) + message = f"API({scan_create_endpoint}) return error code {res.status_code} for workspace id({workspace_id})" LOGGER.warning(message) @@ -740,46 +760,40 @@ def wait_for_scan_to_complete(scan_id: str, timeout: int) -> Boolean: minimum_sleep = 3 if timeout < minimum_sleep: LOGGER.info( - "Setting timeout to minimum_sleep time {} seconds".format( - minimum_sleep - ) + f"Setting timeout to minimum_sleep time {minimum_sleep} seconds" ) timeout = minimum_sleep - max_trial = int(timeout / minimum_sleep) - LOGGER.info("Max trial {}".format(max_trial)) + max_trial = timeout // minimum_sleep + LOGGER.info(f"Max trial {max_trial}") scan_get_endpoint = PowerBiAPI.API_ENDPOINTS[Constant.SCAN_GET] scan_get_endpoint = scan_get_endpoint.format( - POWERBI_ADMIN_BASE_URL=self.__config.admin_base_url, SCAN_ID=scan_id + POWERBI_ADMIN_BASE_URL=PowerBiAPI.ADMIN_BASE_URL, SCAN_ID=scan_id ) - LOGGER.info("Hitting URL={}".format(scan_get_endpoint)) + LOGGER.info(f"Hitting URL={scan_get_endpoint}") trail = 1 while True: - LOGGER.info("Trial = {}".format(trail)) + LOGGER.info(f"Trial = {trail}") res = requests.get( scan_get_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) if res.status_code != 200: - message = "API({}) return error code {} for scan id({})".format( - scan_get_endpoint, res.status_code, scan_id - ) + message = f"API({scan_get_endpoint}) return error code {res.status_code} for scan id({scan_id})" LOGGER.warning(message) raise ConnectionError(message) if res.json()["status"].upper() == "Succeeded".upper(): - LOGGER.info( - "Scan result is available for scan id({})".format(scan_id) - ) + LOGGER.info(f"Scan result is available for scan id({scan_id})") return True if trail == max_trial: break - LOGGER.info("Sleeping for {} seconds".format(minimum_sleep)) + LOGGER.info(f"Sleeping for {minimum_sleep} seconds") sleep(minimum_sleep) trail += 1 @@ -788,23 +802,21 @@ def wait_for_scan_to_complete(scan_id: str, timeout: int) -> Boolean: def get_scan_result(scan_id: str) -> dict: LOGGER.info("Fetching scan result") - LOGGER.info("{}={}".format(Constant.SCAN_ID, scan_id)) + LOGGER.info(f"{Constant.SCAN_ID}={scan_id}") scan_result_get_endpoint = PowerBiAPI.API_ENDPOINTS[ Constant.SCAN_RESULT_GET ] scan_result_get_endpoint = scan_result_get_endpoint.format( - POWERBI_ADMIN_BASE_URL=self.__config.admin_base_url, SCAN_ID=scan_id + POWERBI_ADMIN_BASE_URL=PowerBiAPI.ADMIN_BASE_URL, SCAN_ID=scan_id ) - LOGGER.info("Hittin URL={}".format(scan_result_get_endpoint)) + LOGGER.info(f"Hitting URL={scan_result_get_endpoint}") res = requests.get( scan_result_get_endpoint, headers={Constant.Authorization: self.get_access_token()}, ) if res.status_code != 200: - message = "API({}) return error code {} for scan id({})".format( - scan_result_get_endpoint, res.status_code, scan_id - ) + message = f"API({scan_result_get_endpoint}) return error code {res.status_code} for scan id({scan_id})" LOGGER.warning(message) @@ -821,10 +833,9 @@ def json_to_dataset_map(scan_result: dict) -> dict: if datasets is None or len(datasets) == 0: LOGGER.warning( - "Workspace {}({}) does not have datasets".format( - scan_result["name"], scan_result["id"] - ) + f'Workspace {scan_result["name"]}({scan_result["id"]}) does not have datasets' ) + LOGGER.info("Returning empty datasets") return dataset_map @@ -844,18 +855,15 @@ def json_to_dataset_map(scan_result: dict) -> dict: and dataset_instance.datasource.metadata.is_relational is True ): LOGGER.info( - "Processing tables attribute for dataset {}({})".format( - dataset_instance.name, dataset_instance.id - ) + f"Processing tables attribute for dataset {dataset_instance.name}({dataset_instance.id})" ) for table in dataset_dict["tables"]: if "Value.NativeQuery(" in table["source"][0]["expression"]: LOGGER.warning( - "Table {} is created from Custom SQL. Ignoring in processing".format( - table["name"] - ) + f'Table {table["name"]} is created from Custom SQL. Ignoring in processing' ) + continue # PowerBi table name contains schema name and table name. Format is @@ -892,6 +900,7 @@ def init_dashboard_tiles(workspace: PowerBiAPI.Workspace) -> None: # Scan is complete lets take the result scan_result = get_scan_result(scan_id=scan_id) + LOGGER.debug("scan result = {}".format(scan_result)) workspace = PowerBiAPI.Workspace( id=scan_result["id"], name=scan_result["name"], @@ -970,34 +979,30 @@ def __to_datahub_dataset( if dataset is None: return dataset_mcps - # We are only suporting relation PowerBi DataSources + # We are only supporting relation PowerBi DataSources if ( dataset.datasource is None or dataset.datasource.metadata.is_relational is False ): LOGGER.warning( - "Dataset {}({}) is not created from relational datasource".format( - dataset.name, dataset.id - ) + f"Dataset {dataset.name}({dataset.id}) is not created from relational datasource" ) + return dataset_mcps LOGGER.info( - "Converting dataset={}(id={}) to datahub dataset".format( - dataset.name, dataset.id - ) + f"Converting dataset={dataset.name}(id={dataset.id}) to datahub dataset" ) for table in dataset.tables: # Create an URN for dataset ds_urn = builder.make_dataset_urn( platform=self.__config.dataset_type_mapping[dataset.datasource.type], - name="{}.{}.{}".format( - dataset.datasource.database, table.schema_name, table.name - ), + name=f"{dataset.datasource.database}.{table.schema_name}.{table.name}", env=self.__config.env, ) - LOGGER.info("{}={}".format(Constant.Dataset_URN, ds_urn)) + + LOGGER.info(f"{Constant.Dataset_URN}={ds_urn}") # Create datasetProperties mcp ds_properties = DatasetPropertiesClass(description=table.name) @@ -1166,18 +1171,21 @@ def chart_custom_properties(dashboard: PowerBiAPI.Dashboard) -> dict: # Dashboard Ownership owners = [ - OwnerClass(owner=user_urn, type=OwnershipTypeClass.CONSUMER) + OwnerClass(owner=user_urn, type=OwnershipTypeClass.NONE) for user_urn in user_urn_list if user_urn is not None ] - ownership = OwnershipClass(owners=owners) - # Dashboard owner MCP - owner_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, - entity_urn=dashboard_urn, - aspect_name=Constant.OWNERSHIP, - aspect=ownership, - ) + + owner_mcp = None + if len(owners) > 0: + # Dashboard owner MCP + ownership = OwnershipClass(owners=owners) + owner_mcp = self.new_mcp( + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, + aspect_name=Constant.OWNERSHIP, + aspect=ownership, + ) # Dashboard browsePaths browse_path = BrowsePathsClass( @@ -1190,14 +1198,18 @@ def chart_custom_properties(dashboard: PowerBiAPI.Dashboard) -> dict: aspect=browse_path, ) - return [ + list_of_mcps = [ browse_path_mcp, info_mcp, removed_status_mcp, dashboard_key_mcp, - owner_mcp, ] + if owner_mcp is not None: + list_of_mcps.append(owner_mcp) + + return list_of_mcps + def to_datahub_user( self, user: PowerBiAPI.User ) -> List[MetadataChangeProposalWrapper]: @@ -1206,9 +1218,7 @@ def to_datahub_user( """ LOGGER.info( - "Converting user {}(id={}) to datahub's user".format( - user.displayName, user.id - ) + f"Converting user {user.displayName}(id={user.id}) to datahub's user" ) # Create an URN for user @@ -1266,10 +1276,10 @@ def to_datahub_chart( chart_mcps = [] # Return empty list if input list is empty - if len(tiles) == 0: + if not tiles: return [], [] - LOGGER.info("Converting tiles(count={}) to charts".format(len(tiles))) + LOGGER.info(f"Converting tiles(count={len(tiles)}) to charts") for tile in tiles: if tile is None: @@ -1292,7 +1302,7 @@ def to_datahub_work_units( mcps = [] LOGGER.info( - "Converting dashboard={} to datahub dashboard".format(dashboard.displayName) + f"Converting dashboard={dashboard.displayName} to datahub dashboard" ) # Convert user to CorpUser @@ -1337,23 +1347,15 @@ def report_charts_dropped(self, view: str) -> None: @platform_name("PowerBI") @config_class(PowerBiDashboardSourceConfig) @support_status(SupportStatus.CERTIFIED) -@capability(SourceCapability.OWNERSHIP, "Enabled by default") +@capability( + SourceCapability.OWNERSHIP, "On by default but can disabled by configuration" +) class PowerBiDashboardSource(Source): """ - This plugin extracts the following: - - - Power BI dashboards, tiles, datasets + This plugin extracts the following: + - Power BI dashboards, tiles and datasets - Names, descriptions and URLs of dashboard and tile - Owners of dashboards - - ## Configuration Notes - - See the - 1. [Microsoft AD App Creation doc](https://docs.microsoft.com/en-us/power-bi/developer/embedded/embed-service-principal) for the steps to create a app client ID and secret. - 2. Login to Power BI as Admin and from `Tenant settings` allow below permissions. - - Allow service principles to use Power BI APIs - - Allow service principals to use read-only Power BI admin APIs - - Enhance admin APIs responses with detailed metadata """ source_config: PowerBiDashboardSourceConfig @@ -1391,12 +1393,10 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.reporter.report_dashboards_scanned() self.reporter.report_charts_scanned(count=len(dashboard.tiles)) except Exception as e: - message = "Error ({}) occurred while loading dashboard {}(id={}) tiles.".format( - e, dashboard.displayName, dashboard.id - ) + message = f"Error ({e}) occurred while loading dashboard {dashboard.displayName}(id={dashboard.id}) tiles." + LOGGER.exception(message, e) self.reporter.report_warning(dashboard.id, message) - # Convert PowerBi Dashboard and child entities to Datahub work unit to ingest into Datahub workunits = self.mapper.to_datahub_work_units(dashboard) for workunit in workunits: diff --git a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py index e4d9a505ea7210..ffc0253a070b2d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py +++ b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py @@ -98,7 +98,7 @@ def __init__(self, config: PulsarSourceConfig, ctx: PipelineContext): self.platform: str = "pulsar" self.config: PulsarSourceConfig = config self.report: PulsarSourceReport = PulsarSourceReport() - self.base_url: str = self.config.web_service_url + "/admin/v2" + self.base_url: str = f"{self.config.web_service_url}/admin/v2" self.tenants: List[str] = config.tenants if ( @@ -120,7 +120,7 @@ def __init__(self, config: PulsarSourceConfig, ctx: PipelineContext): if self._is_oauth_authentication_configured(): # Get OpenId configuration from issuer, e.g. token_endpoint oid_config_url = ( - "%s/.well-known/openid-configuration" % self.config.issuer_url + f"{self.config.issuer_url}/.well-known/openid-configuration" ) oid_config_response = requests.get( oid_config_url, verify=False, allow_redirects=False @@ -130,8 +130,7 @@ def __init__(self, config: PulsarSourceConfig, ctx: PipelineContext): self.config.oid_config.update(oid_config_response.json()) else: logger.error( - "Unexpected response while getting discovery document using %s : %s" - % (oid_config_url, oid_config_response) + f"Unexpected response while getting discovery document using {oid_config_url} : {oid_config_response}" ) if "token_endpoint" not in self.config.oid_config: @@ -325,15 +324,14 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # Report the Pulsar broker version we are communicating with self.report.report_pulsar_version( self.session.get( - "%s/brokers/version" % self.base_url, - timeout=self.config.timeout, + f"{self.base_url}/brokers/version", timeout=self.config.timeout ).text ) # If no tenants are provided, request all tenants from cluster using /admin/v2/tenants endpoint. # Requesting cluster tenant information requires superuser privileges if not self.tenants: - self.tenants = self._get_pulsar_metadata(self.base_url + "/tenants") or [] + self.tenants = self._get_pulsar_metadata(f"{self.base_url}/tenants") or [] # Initialize counters self.report.tenants_scanned = 0 @@ -346,9 +344,10 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # Get namespaces belonging to a tenant, /admin/v2/%s/namespaces # A tenant admin role has sufficient privileges to perform this action namespaces = ( - self._get_pulsar_metadata(self.base_url + "/namespaces/%s" % tenant) + self._get_pulsar_metadata(f"{self.base_url}/namespaces/{tenant}") or [] ) + for namespace in namespaces: self.report.namespaces_scanned += 1 if self.config.namespace_patterns.allowed(namespace): @@ -406,14 +405,10 @@ def _add_topic_to_checkpoint(self, topic: str) -> None: ) def _is_token_authentication_configured(self) -> bool: - if self.config.token is not None: - return True - return False + return self.config.token is not None def _is_oauth_authentication_configured(self) -> bool: - if self.config.issuer_url is not None: - return True - return False + return self.config.issuer_url is not None def _get_schema_and_fields( self, pulsar_topic: PulsarTopic, is_key_schema: bool @@ -421,10 +416,9 @@ def _get_schema_and_fields( pulsar_schema: Optional[PulsarSchema] = None - schema_url = self.base_url + "/schemas/%s/%s/%s/schema" % ( - pulsar_topic.tenant, - pulsar_topic.namespace, - pulsar_topic.topic, + schema_url = ( + self.base_url + + f"/schemas/{pulsar_topic.tenant}/{pulsar_topic.namespace}/{pulsar_topic.topic}/schema" ) schema_payload = self._get_pulsar_metadata(schema_url) @@ -449,7 +443,7 @@ def _get_schema_fields( ) -> List[SchemaField]: # Parse the schema and convert it to SchemaFields. fields: List[SchemaField] = [] - if schema.schema_type == "AVRO" or schema.schema_type == "JSON": + if schema.schema_type in ["AVRO", "JSON"]: # Extract fields from schema and get the FQN for the schema fields = schema_util.avro_schema_to_mce_fields( schema.schema_str, is_key_schema=is_key_schema @@ -465,6 +459,7 @@ def _get_schema_metadata( self, pulsar_topic: PulsarTopic, platform_urn: str ) -> Tuple[Optional[PulsarSchema], Optional[SchemaMetadata]]: + # FIXME: Type annotations are not working for this function. schema, fields = self._get_schema_and_fields( pulsar_topic=pulsar_topic, is_key_schema=False ) # type: Tuple[Optional[PulsarSchema], List[SchemaField]] diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index 2abd61849ac260..29bcc5ab73f0c2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -1,4 +1,3 @@ -import importlib import logging import math import sys @@ -22,6 +21,7 @@ platform_name, support_status, ) +from datahub.ingestion.api.registry import import_path from datahub.ingestion.api.source import Source, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import ( @@ -203,29 +203,33 @@ def get_full_qualified_name(self, database_name: str, table_name: str) -> str: def get_full_qualified_name(platform: str, database_name: str, table_name: str) -> str: - if platform == "postgres": - full_qualified_name = PostgresQualifiedNameParser().get_full_qualified_name( + if platform == "athena": + return AthenaQualifiedNameParser().get_full_qualified_name( database_name, table_name ) - elif platform == "mysql": - full_qualified_name = MysqlQualifiedNameParser().get_full_qualified_name( + + elif platform == "bigquery": + return BigqueryQualifiedNameParser().get_full_qualified_name( database_name, table_name ) + elif platform == "mssql": - full_qualified_name = MssqlQualifiedNameParser().get_full_qualified_name( + return MssqlQualifiedNameParser().get_full_qualified_name( database_name, table_name ) - elif platform == "athena": - full_qualified_name = AthenaQualifiedNameParser().get_full_qualified_name( + + elif platform == "mysql": + return MysqlQualifiedNameParser().get_full_qualified_name( database_name, table_name ) - elif platform == "bigquery": - full_qualified_name = BigqueryQualifiedNameParser().get_full_qualified_name( + + elif platform == "postgres": + return PostgresQualifiedNameParser().get_full_qualified_name( database_name, table_name ) + else: - full_qualified_name = f"{database_name}.{table_name}" - return full_qualified_name + return f"{database_name}.{table_name}" class RedashConfig(ConfigModel): @@ -359,7 +363,7 @@ def warn(self, log: logging.Logger, key: str, reason: str) -> None: self.report.report_warning(key, reason) log.warning(f"{key} => {reason}") - def test_connection(self) -> None: + def validate_connection(self) -> None: test_response = self.client._get(f"{self.config.connect_uri}/api") if test_response.status_code == 200: logger.info("Redash API connected succesfully") @@ -374,11 +378,10 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: @classmethod def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]: assert "." in sql_parser_path, "sql_parser-path must contain a ." - module_name, cls_name = sql_parser_path.rsplit(".", 1) - parser_cls = getattr(importlib.import_module(module_name), cls_name) + parser_cls = import_path(sql_parser_path) + if not issubclass(parser_cls, SQLParser): raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}") - return parser_cls @classmethod @@ -405,8 +408,7 @@ def _get_platform_based_on_datasource(self, data_source: Dict) -> str: map = REDASH_DATA_SOURCE_TO_DATAHUB_MAP.get( data_source_type, {"platform": DEFAULT_DATA_SOURCE_PLATFORM} ) - platform = map.get("platform", DEFAULT_DATA_SOURCE_PLATFORM) - return platform + return map.get("platform", DEFAULT_DATA_SOURCE_PLATFORM) return DEFAULT_DATA_SOURCE_PLATFORM def _get_database_name_based_on_datasource( @@ -597,7 +599,7 @@ def _process_dashboard_response( # Tested the same with a Redash instance dashboard_id = dashboard_response["id"] dashboard_data = self.client._get( - "api/dashboards/{}".format(dashboard_id) + f"api/dashboards/{dashboard_id}" ).json() except Exception: # This does not work in our testing but keeping for now because @@ -686,9 +688,7 @@ def _get_chart_snapshot(self, query_data: Dict, viz_data: Dict) -> ChartSnapshot chart_type = self._get_chart_type_from_viz_data(viz_data) query_id = query_data.get("id") chart_url = f"{self.config.connect_uri}/queries/{query_id}#{viz_id}" - description = ( - viz_data.get("description", "") if viz_data.get("description", "") else "" - ) + description = viz_data.get("description", "") or "" data_source_id = query_data.get("data_source_id") data_source = self._get_chart_data_source(data_source_id) data_source_type = data_source.get("type") @@ -768,7 +768,7 @@ def add_config_to_report(self) -> None: self.report.api_page_limit = self.config.api_page_limit def get_workunits(self) -> Iterable[MetadataWorkUnit]: - self.test_connection() + self.validate_connection() self.add_config_to_report() with PerfTimer() as timer: yield from self._emit_chart_mces() diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py index 43fa4a5c4bede5..dfffdcaa9708dc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py @@ -1,181 +1,23 @@ import logging -import os -import re -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional -import parse import pydantic from pydantic.fields import Field -from wcmatch import pathlib -from datahub.configuration.common import AllowDenyPattern, ConfigModel +from datahub.configuration.common import AllowDenyPattern from datahub.configuration.source_common import ( EnvBasedSourceConfigBase, PlatformSourceConfigBase, ) from datahub.ingestion.source.aws.aws_common import AwsSourceConfig -from datahub.ingestion.source.aws.s3_util import get_bucket_name, is_s3_uri +from datahub.ingestion.source.aws.path_spec import PathSpec +from datahub.ingestion.source.aws.s3_util import get_bucket_name from datahub.ingestion.source.s3.profiling import DataLakeProfilerConfig # hide annoying debug errors from py4j logging.getLogger("py4j").setLevel(logging.ERROR) logger: logging.Logger = logging.getLogger(__name__) -SUPPORTED_FILE_TYPES: List[str] = ["csv", "tsv", "json", "parquet", "avro"] -SUPPORTED_COMPRESSIONS: List[str] = ["gz", "bz2"] - - -class PathSpec(ConfigModel): - class Config: - arbitrary_types_allowed = True - - include: str = Field( - description="Path to table (s3 or local file system). Name variable {table} is used to mark the folder with dataset. In absence of {table}, file level dataset will be created. Check below examples for more details." - ) - exclude: Optional[List[str]] = Field( - default=None, - description="list of paths in glob pattern which will be excluded while scanning for the datasets", - ) - file_types: List[str] = Field( - default=SUPPORTED_FILE_TYPES, - description="Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.", - ) - - default_extension: Optional[str] = Field( - description="For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.", - ) - - table_name: Optional[str] = Field( - default=None, - description="Display name of the dataset.Combination of named variableds from include path and strings", - ) - - enable_compression: bool = Field( - default=True, - description="Enable or disable processing compressed files. Currenly .gz and .bz files are supported.", - ) - - sample_files: bool = Field( - default=True, - description="Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled", - ) - - # to be set internally - _parsable_include: str - _compiled_include: parse.Parser - _glob_include: str - _is_s3: bool - - def allowed(self, path: str) -> bool: - logger.debug(f"Checking file to inclusion: {path}") - if not pathlib.PurePath(path).globmatch( - self._glob_include, flags=pathlib.GLOBSTAR - ): - return False - logger.debug(f"{path} matched include ") - if self.exclude: - for exclude_path in self.exclude: - if pathlib.PurePath(path).globmatch( - exclude_path, flags=pathlib.GLOBSTAR - ): - return False - logger.debug(f"{path} is not excluded") - ext = os.path.splitext(path)[1].strip(".") - - if (ext == "" and self.default_extension is None) and ( - ext != "*" and ext not in self.file_types - ): - return False - - logger.debug(f"{path} had selected extension {ext}") - logger.debug(f"{path} allowed for dataset creation") - return True - - def is_s3(self): - return self._is_s3 - - @classmethod - def get_parsable_include(cls, include: str) -> str: - parsable_include = include - for i in range(parsable_include.count("*")): - parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1) - return parsable_include - - def get_named_vars(self, path: str) -> Union[None, parse.Result, parse.Match]: - return self._compiled_include.parse(path) - - @pydantic.root_validator() - def validate_path_spec(cls, values: Dict) -> Dict[str, Any]: - - if "**" in values["include"]: - raise ValueError("path_spec.include cannot contain '**'") - - if values.get("file_types") is None: - values["file_types"] = SUPPORTED_FILE_TYPES - else: - for file_type in values["file_types"]: - if file_type not in SUPPORTED_FILE_TYPES: - raise ValueError( - f"file type {file_type} not in supported file types. Please specify one from {SUPPORTED_FILE_TYPES}" - ) - - if values.get("default_extension") is not None: - if values.get("default_extension") not in SUPPORTED_FILE_TYPES: - raise ValueError( - f"default extension {values.get('default_extension')} not in supported default file extension. Please specify one from {SUPPORTED_FILE_TYPES}" - ) - - include_ext = os.path.splitext(values["include"])[1].strip(".") - if ( - include_ext not in values["file_types"] - and include_ext != "*" - and not values["default_extension"] - and include_ext not in SUPPORTED_COMPRESSIONS - ): - raise ValueError( - f"file type specified ({include_ext}) in path_spec.include is not in specified file " - f'types. Please select one from {values.get("file_types")} or specify ".*" to allow all types' - ) - - values["_parsable_include"] = PathSpec.get_parsable_include(values["include"]) - logger.debug(f'Setting _parsable_include: {values.get("_parsable_include")}') - compiled_include_tmp = parse.compile(values["_parsable_include"]) - values["_compiled_include"] = compiled_include_tmp - logger.debug(f'Setting _compiled_include: {values["_compiled_include"]}') - values["_glob_include"] = re.sub(r"\{[^}]+\}", "*", values["include"]) - logger.debug(f'Setting _glob_include: {values.get("_glob_include")}') - - if values.get("table_name") is None: - if "{table}" in values["include"]: - values["table_name"] = "{table}" - else: - logger.debug(f"include fields: {compiled_include_tmp.named_fields}") - logger.debug( - f"table_name fields: {parse.compile(values['table_name']).named_fields}" - ) - if not all( - x in values["_compiled_include"].named_fields - for x in parse.compile(values["table_name"]).named_fields - ): - raise ValueError( - "Not all named variables used in path_spec.table_name are specified in " - "path_spec.include" - ) - - if values.get("exclude") is not None: - for exclude_path in values["exclude"]: - if len(parse.compile(exclude_path).named_fields) != 0: - raise ValueError( - "path_spec.exclude should not contain any named variables" - ) - - values["_is_s3"] = is_s3_uri(values["include"]) - if not values["_is_s3"]: - # Sampling only makes sense on s3 currently - values["sample_files"] = False - logger.debug(f'Setting _is_s3: {values.get("_is_s3")}') - return values - class DataLakeSourceConfig(PlatformSourceConfigBase, EnvBasedSourceConfigBase): path_specs: Optional[List[PathSpec]] = Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py b/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py index f2806f362ab97c..d1f0b9c625c1cf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py @@ -273,7 +273,8 @@ def __init__( ) column_null_counts = null_counts.toPandas().T[0].to_dict() column_null_fractions = { - c: column_null_counts[c] / self.row_count for c in self.columns_to_profile + c: column_null_counts[c] / self.row_count if self.row_count != 0 else 0 + for c in self.columns_to_profile } column_nonnull_counts = { c: self.row_count - column_null_counts[c] for c in self.columns_to_profile diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index 29f54dc9449e3e..b2a619976353df 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -38,17 +38,8 @@ from datahub.emitter.mce_builder import ( make_data_platform_urn, make_dataset_urn_with_platform_instance, - make_tag_urn, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import ( - FolderKey, - KeyType, - PlatformKey, - S3BucketKey, - add_dataset_to_container, - gen_containers, -) from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -60,12 +51,14 @@ ) from datahub.ingestion.api.source import Source, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.aws.s3_boto_utils import get_s3_tags, list_folders from datahub.ingestion.source.aws.s3_util import ( get_bucket_name, get_bucket_relative_path, get_key_prefix, strip_s3_prefix, ) +from datahub.ingestion.source.data_lake.data_lake_utils import ContainerWUCreator from datahub.ingestion.source.s3.config import DataLakeSourceConfig, PathSpec from datahub.ingestion.source.s3.profiling import _SingleTableProfiler from datahub.ingestion.source.s3.report import DataLakeSourceReport @@ -87,10 +80,8 @@ from datahub.metadata.schema_classes import ( ChangeTypeClass, DatasetPropertiesClass, - GlobalTagsClass, MapTypeClass, OtherSchemaClass, - TagAssociationClass, ) from datahub.telemetry import stats, telemetry from datahub.utilities.perf_timer import PerfTimer @@ -229,7 +220,7 @@ class S3Source(Source): source_config: DataLakeSourceConfig report: DataLakeSourceReport profiling_times_taken: List[float] - processed_containers: List[str] + container_WU_creator: ContainerWUCreator def __init__(self, config: DataLakeSourceConfig, ctx: PipelineContext): super().__init__(ctx) @@ -382,80 +373,6 @@ def read_file_spark(self, file: str, ext: str) -> Optional[DataFrame]: # see https://mungingdata.com/pyspark/avoid-dots-periods-column-names/ return df.toDF(*(c.replace(".", "_") for c in df.columns)) - def create_emit_containers( - self, - container_key: KeyType, - name: str, - sub_types: List[str], - parent_container_key: Optional[PlatformKey] = None, - domain_urn: Optional[str] = None, - ) -> Iterable[MetadataWorkUnit]: - if container_key.guid() not in self.processed_containers: - container_wus = gen_containers( - container_key=container_key, - name=name, - sub_types=sub_types, - parent_container_key=parent_container_key, - domain_urn=domain_urn, - ) - self.processed_containers.append(container_key.guid()) - logger.debug(f"Creating container with key: {container_key}") - for wu in container_wus: - self.report.report_workunit(wu) - yield wu - - def create_container_hierarchy( - self, table_data: TableData, dataset_urn: str - ) -> Iterable[MetadataWorkUnit]: - logger.debug(f"Creating containers for {dataset_urn}") - base_full_path = table_data.table_path - parent_key = None - if table_data.is_s3: - bucket_name = get_bucket_name(table_data.table_path) - bucket_key = self.gen_bucket_key(bucket_name) - yield from self.create_emit_containers( - container_key=bucket_key, - name=bucket_name, - sub_types=["S3 bucket"], - parent_container_key=None, - ) - parent_key = bucket_key - base_full_path = get_bucket_relative_path(table_data.table_path) - - parent_folder_path = ( - base_full_path[: base_full_path.rfind("/")] - if base_full_path.rfind("/") != -1 - else "" - ) - - # Dataset is in the root folder - if not parent_folder_path and parent_key is None: - logger.warning( - f"Failed to associate Dataset ({dataset_urn}) with container" - ) - return - - for folder in parent_folder_path.split("/"): - abs_path = folder - if parent_key: - prefix: str = "" - if isinstance(parent_key, S3BucketKey): - prefix = parent_key.bucket_name - elif isinstance(parent_key, FolderKey): - prefix = parent_key.folder_abs_path - abs_path = prefix + "/" + folder - folder_key = self.gen_folder_key(abs_path) - yield from self.create_emit_containers( - container_key=folder_key, - name=folder, - sub_types=["Folder"], - parent_container_key=parent_key, - ) - parent_key = folder_key - - assert parent_key is not None - yield from add_dataset_to_container(parent_key, dataset_urn) - def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List: if table_data.is_s3: if self.source_config.aws_config is None: @@ -474,7 +391,8 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List: extension = pathlib.Path(table_data.full_path).suffix if path_spec.enable_compression and ( - extension[1:] in datahub.ingestion.source.s3.config.SUPPORTED_COMPRESSIONS + extension[1:] + in datahub.ingestion.source.aws.path_spec.SUPPORTED_COMPRESSIONS ): # Removing the compression extension and using the one before that like .json.gz -> .json extension = pathlib.Path(table_data.full_path).with_suffix("").suffix @@ -519,9 +437,14 @@ def get_table_profile( # read in the whole table with Spark for profiling table = None try: - table = self.read_file_spark( - table_data.table_path, os.path.splitext(table_data.full_path)[1] - ) + if table_data.partitions: + table = self.read_file_spark( + table_data.table_path, os.path.splitext(table_data.full_path)[1] + ) + else: + table = self.read_file_spark( + table_data.full_path, os.path.splitext(table_data.full_path)[1] + ) except Exception as e: logger.error(e) @@ -648,7 +571,15 @@ def ingest_table( if table_data.full_path == table_data.table_path else None ) - s3_tags = self.get_s3_tags(bucket, key_prefix, dataset_urn) + s3_tags = get_s3_tags( + bucket, + key_prefix, + dataset_urn, + self.source_config.aws_config, + self.ctx, + self.source_config.use_s3_bucket_tags, + self.source_config.use_s3_object_tags, + ) if s3_tags is not None: dataset_snapshot.aspects.append(s3_tags) @@ -657,88 +588,16 @@ def ingest_table( self.report.report_workunit(wu) yield wu - yield from self.create_container_hierarchy(table_data, dataset_urn) + container_wus = self.container_WU_creator.create_container_hierarchy( + table_data.table_path, table_data.is_s3, dataset_urn + ) + for wu in container_wus: + self.report.report_workunit(wu) + yield wu if self.source_config.profiling.enabled: yield from self.get_table_profile(table_data, dataset_urn) - def gen_bucket_key(self, name): - return S3BucketKey( - platform="s3", - instance=self.source_config.env - if self.source_config.platform_instance is None - else self.source_config.platform_instance, - bucket_name=name, - ) - - def get_s3_tags( - self, bucket_name: str, key_name: Optional[str], dataset_urn: str - ) -> Optional[GlobalTagsClass]: - if self.source_config.aws_config is None: - raise ValueError("aws_config not set. Cannot browse s3") - new_tags = GlobalTagsClass(tags=[]) - tags_to_add = [] - if self.source_config.use_s3_bucket_tags: - s3 = self.source_config.aws_config.get_s3_resource() - bucket = s3.Bucket(bucket_name) - try: - tags_to_add.extend( - [ - make_tag_urn(f"""{tag["Key"]}:{tag["Value"]}""") - for tag in bucket.Tagging().tag_set - ] - ) - except s3.meta.client.exceptions.ClientError: - logger.warn(f"No tags found for bucket={bucket_name}") - - if self.source_config.use_s3_object_tags and key_name is not None: - s3_client = self.source_config.aws_config.get_s3_client() - object_tagging = s3_client.get_object_tagging( - Bucket=bucket_name, Key=key_name - ) - tag_set = object_tagging["TagSet"] - if tag_set: - tags_to_add.extend( - [ - make_tag_urn(f"""{tag["Key"]}:{tag["Value"]}""") - for tag in tag_set - ] - ) - else: - # Unlike bucket tags, if an object does not have tags, it will just return an empty array - # as opposed to an exception. - logger.warn(f"No tags found for bucket={bucket_name} key={key_name}") - if len(tags_to_add) == 0: - return None - if self.ctx.graph is not None: - logger.debug("Connected to DatahubApi, grabbing current tags to maintain.") - current_tags: Optional[GlobalTagsClass] = self.ctx.graph.get_aspect_v2( - entity_urn=dataset_urn, - aspect="globalTags", - aspect_type=GlobalTagsClass, - ) - if current_tags: - tags_to_add.extend( - [current_tag.tag for current_tag in current_tags.tags] - ) - else: - logger.warn("Could not connect to DatahubApi. No current tags to maintain") - # Remove duplicate tags - tags_to_add = list(set(tags_to_add)) - new_tags = GlobalTagsClass( - tags=[TagAssociationClass(tag_to_add) for tag_to_add in tags_to_add] - ) - return new_tags - - def gen_folder_key(self, abs_path): - return FolderKey( - platform=self.source_config.platform, - instance=self.source_config.env - if self.source_config.platform_instance is None - else self.source_config.platform_instance, - folder_abs_path=abs_path, - ) - def get_prefix(self, relative_path: str) -> str: index = re.search(r"[\*|\{]", relative_path) if index: @@ -756,35 +615,18 @@ def extract_table_data( ) -> TableData: logger.debug(f"Getting table data for path: {path}") - parsed_vars = path_spec.get_named_vars(path) + table_name, table_path = path_spec.extract_table_name_and_path(path) table_data = None - if parsed_vars is None or "table" not in parsed_vars.named: - table_data = TableData( - display_name=os.path.basename(path), - is_s3=path_spec.is_s3(), - full_path=path, - partitions=None, - timestamp=timestamp, - table_path=path, - number_of_files=1, - size_in_bytes=size, - ) - else: - include = path_spec.include - depth = include.count("/", 0, include.find("{table}")) - table_path = ( - "/".join(path.split("/")[:depth]) + "/" + parsed_vars.named["table"] - ) - table_data = TableData( - display_name=self.extract_table_name(path_spec, parsed_vars.named), - is_s3=path_spec.is_s3(), - full_path=path, - partitions=None, - timestamp=timestamp, - table_path=table_path, - number_of_files=1, - size_in_bytes=size, - ) + table_data = TableData( + display_name=table_name, + is_s3=path_spec.is_s3(), + full_path=path, + partitions=None, + timestamp=timestamp, + table_path=table_path, + number_of_files=1, + size_in_bytes=size, + ) return table_data def resolve_templated_folders(self, bucket_name: str, prefix: str) -> Iterable[str]: @@ -794,25 +636,14 @@ def resolve_templated_folders(self, bucket_name: str, prefix: str) -> Iterable[s yield prefix return - folders: Iterable[str] = self.list_folders(bucket_name, folder_split[0]) + folders: Iterable[str] = list_folders( + bucket_name, folder_split[0], self.source_config.aws_config + ) for folder in folders: yield from self.resolve_templated_folders( bucket_name, f"{folder}{folder_split[1]}" ) - def list_folders(self, bucket_name: str, prefix: str) -> Iterable[str]: - assert self.source_config.aws_config - s3_client = self.source_config.aws_config.get_s3_client() - paginator = s3_client.get_paginator("list_objects_v2") - for page in paginator.paginate( - Bucket=bucket_name, Prefix=prefix, Delimiter="/" - ): - for o in page.get("CommonPrefixes", []): - folder: str = str(o.get("Prefix")) - if folder.endswith("/"): - folder = folder[:-1] - yield f"{folder}" - def s3_browser(self, path_spec: PathSpec) -> Iterable[Tuple[str, datetime, int]]: if self.source_config.aws_config is None: raise ValueError("aws_config not set. Cannot browse s3") @@ -848,7 +679,9 @@ def s3_browser(self, path_spec: PathSpec) -> Iterable[Tuple[str, datetime, int]] for folder in self.resolve_templated_folders( bucket_name, get_bucket_relative_path(include[:table_index]) ): - for f in self.list_folders(bucket_name, f"{folder}"): + for f in list_folders( + bucket_name, f"{folder}", self.source_config.aws_config + ): logger.info(f"Processing folder: {f}") for obj in ( @@ -886,7 +719,11 @@ def local_browser(self, path_spec: PathSpec) -> Iterable[Tuple[str, datetime, in ), os.path.getsize(full_path) def get_workunits(self) -> Iterable[MetadataWorkUnit]: - self.processed_containers = [] + self.container_WU_creator = ContainerWUCreator( + self.source_config.platform, + self.source_config.platform_instance, + self.source_config.env, + ) with PerfTimer() as timer: assert self.source_config.path_specs for path_spec in self.source_config.path_specs: diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py new file mode 100644 index 00000000000000..cfbff4b018f549 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py @@ -0,0 +1,736 @@ +import json +import logging +import time +from datetime import datetime +from enum import Enum +from typing import Dict, Iterable, List, Optional + +import requests +from pydantic import Field, validator +from simple_salesforce import Salesforce + +import datahub.emitter.mce_builder as builder +from datahub.configuration.common import ( + AllowDenyPattern, + ConfigModel, + ConfigurationError, +) +from datahub.configuration.source_common import DatasetSourceConfigBase +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_domain_to_entity_wu +from datahub.ingestion.api.common import PipelineContext, WorkUnit +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.metadata.schema_classes import ( + AuditStampClass, + BooleanTypeClass, + BytesTypeClass, + ChangeTypeClass, + DataPlatformInstanceClass, + DatasetProfileClass, + DatasetPropertiesClass, + DateTypeClass, + EnumTypeClass, + ForeignKeyConstraintClass, + GlobalTagsClass, + NullTypeClass, + NumberTypeClass, + OperationClass, + OperationTypeClass, + OtherSchemaClass, + RecordTypeClass, + SchemaFieldClass, + SchemaFieldDataTypeClass, + SchemaMetadataClass, + StringTypeClass, + SubTypesClass, + TagAssociationClass, +) +from datahub.utilities import config_clean + +logger = logging.getLogger(__name__) + + +class SalesforceAuthType(Enum): + USERNAME_PASSWORD = "USERNAME_PASSWORD" + DIRECT_ACCESS_TOKEN = "DIRECT_ACCESS_TOKEN" + + +class SalesforceProfilingConfig(ConfigModel): + enabled: bool = Field( + default=False, + description="Whether profiling should be done. Supports only table-level profiling at this stage", + ) + + # TODO - support field level profiling + + +class SalesforceConfig(DatasetSourceConfigBase): + platform = "salesforce" + + auth: SalesforceAuthType = SalesforceAuthType.USERNAME_PASSWORD + + # Username, Password Auth + username: Optional[str] = Field(description="Salesforce username") + password: Optional[str] = Field(description="Password for Salesforce user") + security_token: Optional[str] = Field( + description="Security token for Salesforce username" + ) + # client_id, client_secret not required + + # Direct - Instance URL, Access Token Auth + instance_url: Optional[str] = Field( + description="Salesforce instance url. e.g. https://MyDomainName.my.salesforce.com" + ) + access_token: Optional[str] = Field(description="Access token for instance url") + + ingest_tags: Optional[bool] = Field( + default=False, + description="Ingest Tags from source. This will override Tags entered from UI", + ) + + object_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for Salesforce objects to filter in ingestion.", + ) + domain: Dict[str, AllowDenyPattern] = Field( + default=dict(), + description='Regex patterns for tables/schemas to describe domain_key domain key (domain_key can be any string like "sales".) There can be multiple domain keys specified.', + ) + + profiling: SalesforceProfilingConfig = SalesforceProfilingConfig() + + profile_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for profiles to filter in ingestion, allowed by the `object_pattern`.", + ) + + @validator("instance_url") + def remove_trailing_slash(cls, v): + return config_clean.remove_trailing_slashes(v) + + +class SalesforceSourceReport(SourceReport): + filtered: List[str] = [] + + def report_dropped(self, ent_name: str) -> None: + self.filtered.append(ent_name) + + +# https://developer.salesforce.com/docs/atlas.en-us.api.meta/api/sforce_api_calls_describesObjects_describesObjectresult.htm#FieldType +FIELD_TYPE_MAPPING = { + "string": StringTypeClass, + "boolean": BooleanTypeClass, + "int": NumberTypeClass, + "integer": NumberTypeClass, + "long": NumberTypeClass, + "double": NumberTypeClass, + "date": DateTypeClass, + "datetime": DateTypeClass, + "time": DateTypeClass, + "id": StringTypeClass, # Primary Key + "picklist": EnumTypeClass, + "address": RecordTypeClass, + "location": RecordTypeClass, + "reference": StringTypeClass, # Foreign Key + "currency": NumberTypeClass, + "textarea": StringTypeClass, + "percent": NumberTypeClass, + "phone": StringTypeClass, + "url": StringTypeClass, + "email": StringTypeClass, + "combobox": StringTypeClass, + "multipicklist": StringTypeClass, + "base64": BytesTypeClass, + "anyType": NullTypeClass, + "encryptedstring": StringTypeClass, +} + + +@platform_name("Salesforce") +@config_class(SalesforceConfig) +@support_status(SupportStatus.INCUBATING) +@capability( + capability_name=SourceCapability.PLATFORM_INSTANCE, + description="Can be equivalent to Salesforce organization", +) +@capability( + capability_name=SourceCapability.DOMAINS, + description="Supported via the `domain` config field", +) +@capability( + capability_name=SourceCapability.DATA_PROFILING, + description="Only table level profiling is supported via `profiling.enabled` config field", +) +@capability( + capability_name=SourceCapability.DELETION_DETECTION, + description="Not supported yet", + supported=False, +) +class SalesforceSource(Source): + + base_url: str + config: SalesforceConfig + report: SalesforceSourceReport + session: requests.Session + sf: Salesforce + fieldCounts: Dict[str, int] + + def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None: + super().__init__(ctx) + self.config = config + self.report = SalesforceSourceReport() + self.session = requests.Session() + self.platform: str = "salesforce" + self.fieldCounts = {} + + try: + if self.config.auth is SalesforceAuthType.DIRECT_ACCESS_TOKEN: + logger.debug("Access Token Provided in Config") + assert ( + self.config.access_token is not None + ), "Config access_token is required for DIRECT_ACCESS_TOKEN auth" + assert ( + self.config.instance_url is not None + ), "Config instance_url is required for DIRECT_ACCESS_TOKEN auth" + + self.sf = Salesforce( + instance_url=self.config.instance_url, + session_id=self.config.access_token, + session=self.session, + ) + elif self.config.auth is SalesforceAuthType.USERNAME_PASSWORD: + logger.debug("Username/Password Provided in Config") + assert ( + self.config.username is not None + ), "Config username is required for USERNAME_PASSWORD auth" + assert ( + self.config.password is not None + ), "Config password is required for USERNAME_PASSWORD auth" + assert ( + self.config.security_token is not None + ), "Config security_token is required for USERNAME_PASSWORD auth" + + self.sf = Salesforce( + username=self.config.username, + password=self.config.password, + security_token=self.config.security_token, + session=self.session, + ) + + except Exception as e: + logger.error(e) + raise ConfigurationError("Salesforce login failed") from e + else: + # List all REST API versions and use latest one + versions_url = "https://{instance}/services/data/".format( + instance=self.sf.sf_instance, + ) + versions_response = self.sf._call_salesforce("GET", versions_url).json() + latest_version = versions_response[-1] + version = latest_version["version"] + self.sf.sf_version = version + + self.base_url = "https://{instance}/services/data/v{sf_version}/".format( + instance=self.sf.sf_instance, sf_version=version + ) + + logger.debug( + "Using Salesforce REST API with {label} version: {version}".format( + label=latest_version["label"], version=latest_version["version"] + ) + ) + + def get_workunits(self) -> Iterable[WorkUnit]: + + sObjects = self.get_salesforce_objects() + + for sObject in sObjects: + yield from self.get_salesforce_object_workunits(sObject) + + def get_salesforce_object_workunits(self, sObject: dict) -> Iterable[WorkUnit]: + + sObjectName = sObject["QualifiedApiName"] + + if not self.config.object_pattern.allowed(sObjectName): + self.report.report_dropped(sObjectName) + logger.debug( + "Skipping {sObject}, as it is not allowed by object_pattern".format( + sObject=sObjectName + ) + ) + return + + datasetUrn = builder.make_dataset_urn_with_platform_instance( + self.platform, + sObjectName, + self.config.platform_instance, + self.config.env, + ) + + customObject = {} + if sObjectName.endswith("__c"): # Is Custom Object + customObject = self.get_custom_object_details(sObject["DeveloperName"]) + + # Table Created, LastModified is available for Custom Object + yield from self.get_operation_workunit(customObject, datasetUrn) + + yield self.get_properties_workunit(sObject, customObject, datasetUrn) + + yield from self.get_schema_metadata_workunit( + sObjectName, sObject, customObject, datasetUrn + ) + + yield self.get_subtypes_workunit(sObjectName, datasetUrn) + + if self.config.platform_instance is not None: + yield self.get_platform_instance_workunit(datasetUrn) + + if self.config.domain is not None: + yield from self.get_domain_workunit(sObjectName, datasetUrn) + + if self.config.profiling.enabled and self.config.profile_pattern.allowed( + sObjectName + ): + yield from self.get_profile_workunit(sObjectName, datasetUrn) + + def get_custom_object_details(self, sObjectDeveloperName: str) -> dict: + customObject = {} + query_url = ( + self.base_url + + "tooling/query/?q=SELECT Description, Language, ManageableState, " + + "CreatedDate, CreatedBy.Username, LastModifiedDate, LastModifiedBy.Username " + + "FROM CustomObject where DeveloperName='{0}'".format(sObjectDeveloperName) + ) + custom_objects_response = self.sf._call_salesforce("GET", query_url).json() + if len(custom_objects_response["records"]) > 0: + logger.debug("Salesforce CustomObject query returned with details") + customObject = custom_objects_response["records"][0] + return customObject + + def get_salesforce_objects(self) -> List: + + # Using Describe Global REST API returns many more objects than required. + # Response does not have the attribute ("customizable") that can be used + # to filter out entities not on ObjectManager UI. Hence SOQL on EntityDefinition + # object is used instead, as suggested by salesforce support. + + query_url = ( + self.base_url + + "tooling/query/?q=SELECT DurableId,QualifiedApiName,DeveloperName," + + "Label,PluralLabel,InternalSharingModel,ExternalSharingModel,DeploymentStatus " + + "FROM EntityDefinition WHERE IsCustomizable = true" + ) + entities_response = self.sf._call_salesforce("GET", query_url).json() + logger.debug( + "Salesforce EntityDefinition query returned {count} sObjects".format( + count=len(entities_response["records"]) + ) + ) + return entities_response["records"] + + def get_domain_workunit( + self, dataset_name: str, datasetUrn: str + ) -> Iterable[WorkUnit]: + domain_urn: Optional[str] = None + + for domain, pattern in self.config.domain.items(): + if pattern.allowed(dataset_name): + domain_urn = builder.make_domain_urn(domain) + + if domain_urn: + yield from add_domain_to_entity_wu( + domain_urn=domain_urn, entity_type="dataset", entity_urn=datasetUrn + ) + + def get_platform_instance_workunit(self, datasetUrn: str) -> WorkUnit: + dataPlatformInstance = DataPlatformInstanceClass( + builder.make_data_platform_urn(self.platform), + instance=builder.make_dataplatform_instance_urn( + self.platform, self.config.platform_instance # type:ignore + ), + ) + return self.wrap_aspect_as_workunit( + "dataset", datasetUrn, "dataPlatformInstance", dataPlatformInstance + ) + + def get_operation_workunit( + self, customObject: dict, datasetUrn: str + ) -> Iterable[WorkUnit]: + + if customObject.get("CreatedBy") and customObject.get("CreatedDate"): + timestamp = self.get_time_from_salesforce_timestamp( + customObject["CreatedDate"] + ) + operation = OperationClass( + timestampMillis=timestamp, + operationType=OperationTypeClass.CREATE, + lastUpdatedTimestamp=timestamp, + actor=builder.make_user_urn(customObject["CreatedBy"]["Username"]), + ) + yield self.wrap_aspect_as_workunit( + "dataset", datasetUrn, "operation", operation + ) + + # Note - Object Level LastModified captures changes at table level metadata e.g. table + # description and does NOT capture field level metadata e.g. new field added, existing + # field updated + + if customObject.get("LastModifiedBy") and customObject.get( + "LastModifiedDate" + ): + timestamp = self.get_time_from_salesforce_timestamp( + customObject["LastModifiedDate"] + ) + operation = OperationClass( + timestampMillis=timestamp, + operationType=OperationTypeClass.ALTER, + lastUpdatedTimestamp=timestamp, + actor=builder.make_user_urn( + customObject["LastModifiedBy"]["Username"] + ), + ) + yield self.wrap_aspect_as_workunit( + "dataset", datasetUrn, "operation", operation + ) + + def get_time_from_salesforce_timestamp(self, date: str) -> int: + return round( + datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp() * 1000 + ) + + def get_properties_workunit( + self, sObject: dict, customObject: Dict[str, str], datasetUrn: str + ) -> WorkUnit: + propertyLabels = { + # from EntityDefinition + "DurableId": "Durable Id", + "DeveloperName": "Developer Name", + "QualifiedApiName": "Qualified API Name", + "Label": "Label", + "PluralLabel": "Plural Label", + "InternalSharingModel": "Internal Sharing Model", + "ExternalSharingModel": "External Sharing Model", + # from CustomObject + "ManageableState": "Manageable State", + "Language": "Language", + } + + sObjectProperties = { + propertyLabels[k]: str(v) + for k, v in sObject.items() + if k in propertyLabels and v is not None + } + sObjectProperties.update( + { + propertyLabels[k]: str(v) + for k, v in customObject.items() + if k in propertyLabels and v is not None + } + ) + + datasetProperties = DatasetPropertiesClass( + name=sObject["Label"], + description=customObject.get("Description"), + customProperties=sObjectProperties, + ) + return self.wrap_aspect_as_workunit( + "dataset", datasetUrn, "datasetProperties", datasetProperties + ) + + def get_subtypes_workunit(self, sObjectName: str, datasetUrn: str) -> WorkUnit: + subtypes = [] + if sObjectName.endswith("__c"): + subtypes.append("Custom Object") + else: + subtypes.append("Standard Object") + + return self.wrap_aspect_as_workunit( + entityName="dataset", + entityUrn=datasetUrn, + aspectName="subTypes", + aspect=SubTypesClass(typeNames=subtypes), + ) + + def get_profile_workunit( + self, sObjectName: str, datasetUrn: str + ) -> Iterable[WorkUnit]: + # Here approximate record counts as returned by recordCount API are used as rowCount + # In future, count() SOQL query may be used instead, if required, might be more expensive + sObject_records_count_url = ( + f"{self.base_url}limits/recordCount?sObjects={sObjectName}" + ) + + sObject_record_count_response = self.sf._call_salesforce( + "GET", sObject_records_count_url + ).json() + + logger.debug( + "Received Salesforce {sObject} record count response".format( + sObject=sObjectName + ) + ) + + for entry in sObject_record_count_response.get("sObjects", []): + datasetProfile = DatasetProfileClass( + timestampMillis=int(time.time() * 1000), + rowCount=entry["count"], + columnCount=self.fieldCounts[sObjectName], + ) + yield self.wrap_aspect_as_workunit( + "dataset", datasetUrn, "datasetProfile", datasetProfile + ) + + # Here field description is created from label, description and inlineHelpText + def _get_field_description(self, field: dict, customField: dict) -> str: + desc = field["Label"] + if field.get("FieldDefinition", {}).get("Description"): + desc = "{0}\n\n{1}".format(desc, field["FieldDefinition"]["Description"]) + if field.get("InlineHelpText"): + desc = "{0}\n\n{1}".format(desc, field["InlineHelpText"]) + return desc + + # Here jsonProps is used to add additional salesforce field level properties. + def _get_field_json_props(self, field: dict, customField: dict) -> str: + jsonProps = {} + + if field.get("IsUnique"): + jsonProps["IsUnique"] = True + + return json.dumps(jsonProps) + + def _get_schema_field( + self, + sObjectName: str, + fieldName: str, + fieldType: str, + field: dict, + customField: dict, + ) -> SchemaFieldClass: + fieldPath = fieldName + + TypeClass = FIELD_TYPE_MAPPING.get(fieldType) + if TypeClass is None: + self.report.report_warning( + sObjectName, + f"Unable to map type {fieldType} to metadata schema", + ) + TypeClass = NullTypeClass + + fieldTags: List[str] = self.get_field_tags(fieldName, field) + + schemaField = SchemaFieldClass( + fieldPath=fieldPath, + type=SchemaFieldDataTypeClass(type=TypeClass()), # type:ignore + description=self._get_field_description(field, customField), + # nativeDataType is set to data type shown on salesforce user interface, + # not the corresponding API data type names. + nativeDataType=field["FieldDefinition"]["DataType"], + nullable=field["IsNillable"], + globalTags=get_tags(fieldTags) if self.config.ingest_tags else None, + jsonProps=self._get_field_json_props(field, customField), + ) + + # Created and LastModified Date and Actor are available for Custom Fields only + if customField.get("CreatedDate") and customField.get("CreatedBy"): + schemaField.created = self.get_audit_stamp( + customField["CreatedDate"], customField["CreatedBy"]["Username"] + ) + if customField.get("LastModifiedDate") and customField.get("LastModifiedBy"): + schemaField.lastModified = self.get_audit_stamp( + customField["LastModifiedDate"], + customField["LastModifiedBy"]["Username"], + ) + + return schemaField + + def get_field_tags(self, fieldName: str, field: dict) -> List[str]: + fieldTags: List[str] = [] + + if fieldName.endswith("__c"): + fieldTags.append("Custom") + + # https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/system_fields.htm + sfSystemFields = [ + "Id", + "IsDeleted", + "CreatedById", + "CreatedDate", + "LastModifiedById", + "LastModifiedDate", + "SystemModstamp", + ] + + if fieldName in sfSystemFields: + fieldTags.append("SystemField") + + if field["FieldDefinition"]["ComplianceGroup"] is not None: + # CCPA, COPPA, GDPR, HIPAA, PCI, PersonalInfo, PII + fieldTags.extend( + iter(field["FieldDefinition"]["ComplianceGroup"].split(";")) + ) + return fieldTags + + def get_audit_stamp(self, date: str, username: str) -> AuditStampClass: + return AuditStampClass( + time=self.get_time_from_salesforce_timestamp(date), + actor=builder.make_user_urn(username), + ) + + def get_schema_metadata_workunit( + self, sObjectName: str, sObject: dict, customObject: dict, datasetUrn: str + ) -> Iterable[WorkUnit]: + + sObject_fields_query_url = ( + self.base_url + + "tooling/query?q=SELECT " + + "QualifiedApiName,DeveloperName,Label, FieldDefinition.DataType, DataType," + + "FieldDefinition.LastModifiedDate, FieldDefinition.LastModifiedBy.Username," + + "Precision, Scale, Length, Digits ,FieldDefinition.IsIndexed, IsUnique," + + "IsCompound, IsComponent, ReferenceTo, FieldDefinition.ComplianceGroup," + + "RelationshipName, IsNillable, FieldDefinition.Description, InlineHelpText " + + "FROM EntityParticle WHERE EntityDefinitionId='{0}'".format( + sObject["DurableId"] + ) + ) + + sObject_fields_response = self.sf._call_salesforce( + "GET", sObject_fields_query_url + ).json() + + logger.debug( + "Received Salesforce {sObject} fields response".format(sObject=sObjectName) + ) + + sObject_custom_fields_query_url = ( + self.base_url + + "tooling/query?q=SELECT " + + "DeveloperName,CreatedDate,CreatedBy.Username,InlineHelpText," + + "LastModifiedDate,LastModifiedBy.Username " + + "FROM CustomField WHERE EntityDefinitionId='{0}'".format( + sObject["DurableId"] + ) + ) + + sObject_custom_fields_response = self.sf._call_salesforce( + "GET", sObject_custom_fields_query_url + ).json() + + logger.debug( + "Received Salesforce {sObject} custom fields response".format( + sObject=sObjectName + ) + ) + customFields: Dict[str, Dict] = { + record["DeveloperName"]: record + for record in sObject_custom_fields_response["records"] + } + + fields: List[SchemaFieldClass] = [] + primaryKeys: List[str] = [] + foreignKeys: List[ForeignKeyConstraintClass] = [] + + for field in sObject_fields_response["records"]: + + customField = customFields.get(field["DeveloperName"], {}) + + fieldName = field["QualifiedApiName"] + fieldType = field["DataType"] + + # Skip compound fields. All Leaf fields are ingested instead. + if fieldType in ("address", "location"): + continue + + schemaField: SchemaFieldClass = self._get_schema_field( + sObjectName, fieldName, fieldType, field, customField + ) + fields.append(schemaField) + + if fieldType == "id": + primaryKeys.append(fieldName) + + if ( + fieldType == "reference" + and field["ReferenceTo"]["referenceTo"] is not None + ): + foreignKeys.extend( + list(self.get_foreign_keys_from_field(fieldName, field, datasetUrn)) + ) + + schemaMetadata = SchemaMetadataClass( + schemaName="", + platform=builder.make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=OtherSchemaClass(rawSchema=""), + fields=fields, + primaryKeys=primaryKeys, + foreignKeys=foreignKeys or None, + ) + + # Created Date and Actor are available for Custom Object only + if customObject.get("CreatedDate") and customObject.get("CreatedBy"): + schemaMetadata.created = self.get_audit_stamp( + customObject["CreatedDate"], customObject["CreatedBy"]["Username"] + ) + self.fieldCounts[sObjectName] = len(fields) + + yield self.wrap_aspect_as_workunit( + "dataset", datasetUrn, "schemaMetadata", schemaMetadata + ) + + def get_foreign_keys_from_field( + self, fieldName: str, field: dict, datasetUrn: str + ) -> Iterable[ForeignKeyConstraintClass]: + # https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/field_types.htm#i1435823 + foreignDatasets = [ + builder.make_dataset_urn_with_platform_instance( + self.platform, + fsObject, + self.config.platform_instance, + self.config.env, + ) + for fsObject in field["ReferenceTo"]["referenceTo"] + ] + + for foreignDataset in foreignDatasets: + yield ForeignKeyConstraintClass( + name=field["RelationshipName"] if field.get("RelationshipName") else "", + foreignDataset=foreignDataset, + foreignFields=[builder.make_schema_field_urn(foreignDataset, "Id")], + sourceFields=[builder.make_schema_field_urn(datasetUrn, fieldName)], + ) + + def wrap_aspect_as_workunit( + self, entityName: str, entityUrn: str, aspectName: str, aspect: builder.Aspect + ) -> WorkUnit: + wu = MetadataWorkUnit( + id=f"{aspectName}-for-{entityUrn}", + mcp=MetadataChangeProposalWrapper( + entityType=entityName, + entityUrn=entityUrn, + aspectName=aspectName, + aspect=aspect, + changeType=ChangeTypeClass.UPSERT, + ), + ) + self.report.report_workunit(wu) + return wu + + def get_report(self) -> SourceReport: + return self.report + + +def get_tags(params: List[str] = None) -> GlobalTagsClass: + if params is None: + params = [] + tags = [TagAssociationClass(tag=builder.make_tag_urn(tag)) for tag in params if tag] + return GlobalTagsClass(tags=tags) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 299ce9f5247a19..10396c1e3c4aed 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -42,12 +42,23 @@ class AthenaConfig(SQLAlchemyConfig): aws_region: str = pydantic.Field( description="Aws region where your Athena database is located" ) + aws_role_arn: Optional[str] = pydantic.Field( + default=None, + description="AWS Role arn for Pyathena to assume in its connection", + ) + aws_role_assumption_duration: int = pydantic.Field( + default=3600, + description="Duration to assume the AWS Role for. Maximum of 43200 (12 hours)", + ) s3_staging_dir: str = pydantic.Field( description="Staging s3 location where the Athena query results will be stored" ) work_group: str = pydantic.Field( description="The name of your Amazon Athena Workgroups" ) + catalog_name: str = pydantic.Field( + default="awsdatacatalog", description="Athena Catalog Name" + ) include_views = False # not supported for Athena @@ -61,6 +72,9 @@ def get_sql_alchemy_url(self): uri_opts={ "s3_staging_dir": self.s3_staging_dir, "work_group": self.work_group, + "catalog_name": self.catalog_name, + "role_arn": self.aws_role_arn, + "duration_seconds": str(self.aws_role_assumption_duration), }, ) @@ -100,7 +114,7 @@ def create(cls, config_dict, ctx): def get_table_properties( self, inspector: Inspector, schema: str, table: str - ) -> Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]]: + ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: if not self.cursor: self.cursor = inspector.dialect._raw_connection(inspector.engine).cursor() diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/sql/bigquery.py index a9d6e2ee413523..c80090b205febb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/bigquery.py @@ -17,6 +17,7 @@ from google.cloud.logging_v2.client import Client as GCPLoggingClient from ratelimiter import RateLimiter from sqlalchemy import create_engine, inspect +from sqlalchemy.engine import Engine from sqlalchemy.engine.reflection import Inspector from datahub.emitter import mce_builder @@ -36,6 +37,7 @@ support_status, ) from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.ge_data_profiler import DatahubGEProfiler from datahub.ingestion.source.sql.sql_common import ( SQLAlchemyConfig, SQLAlchemySource, @@ -152,14 +154,18 @@ c.column_name """.strip() -SHARDED_TABLE_REGEX = r"^(.+)[_](\d{4}|\d{6}|\d{8}|\d{10})$" - BQ_GET_LATEST_SHARD = """ SELECT SUBSTR(MAX(table_id), LENGTH('{table}_') + 1) as max_shard FROM `{project_id}.{schema}.__TABLES_SUMMARY__` WHERE table_id LIKE '{table}%' """.strip() +BQ_GET_LATEST_DATE_TABLE = """ +SELECT MAX(table_name) as max_shard +FROM `{project_id}.{schema}.INFORMATION_SCHEMA.TABLES` +where REGEXP_CONTAINS(table_name, r'^\\d{{{date_length}}}$') +""".strip() + # The existing implementation of this method can be found here: # https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/sqlalchemy_bigquery/base.py#L1018-L1025. @@ -334,23 +340,42 @@ def __init__(self, config, ctx): self.partition_info: Dict[str, str] = dict() atexit.register(cleanup, config) - def get_db_name(self, inspector: Inspector = None) -> str: - if self.config.project_id: + def get_multiproject_project_id( + self, inspector: Optional[Inspector] = None, run_on_compute: bool = False + ) -> Optional[str]: + """ + Use run_on_compute = true when running queries on storage project + where you don't have job create rights + """ + if self.config.storage_project_id and (not run_on_compute): + return self.config.storage_project_id + elif self.config.project_id: return self.config.project_id else: - return self._get_project_id(inspector) + if inspector: + return self._get_project_id(inspector) + else: + return None + + def get_db_name(self, inspector: Inspector) -> str: + """ + DO NOT USE this to get project name when running queries. + That can cause problems with multi-project setups. + Use get_multiproject_project_id with run_on_compute = True + """ + db_name = self.get_multiproject_project_id(inspector) + # db name can't be empty here as we pass in inpector to get_multiproject_project_id + assert db_name + return db_name def _compute_big_query_lineage(self) -> None: if not self.config.include_table_lineage: return - lineage_client_project_id = self._get_lineage_client_project_id() if self.config.use_exported_bigquery_audit_metadata: - self._compute_bigquery_lineage_via_exported_bigquery_audit_metadata( - lineage_client_project_id - ) + self._compute_bigquery_lineage_via_exported_bigquery_audit_metadata() else: - self._compute_bigquery_lineage_via_gcp_logging(lineage_client_project_id) + self._compute_bigquery_lineage_via_gcp_logging() if self.lineage_metadata is None: self.lineage_metadata = {} @@ -361,14 +386,11 @@ def _compute_big_query_lineage(self) -> None: ) logger.debug(f"lineage metadata is {self.lineage_metadata}") - def _compute_bigquery_lineage_via_gcp_logging( - self, lineage_client_project_id: Optional[str] - ) -> None: + def _compute_bigquery_lineage_via_gcp_logging(self) -> None: + project_id = self.get_multiproject_project_id() logger.info("Populating lineage info via GCP audit logs") try: - _clients: List[GCPLoggingClient] = self._make_bigquery_client( - lineage_client_project_id - ) + _clients: List[GCPLoggingClient] = self._make_gcp_logging_client(project_id) template: str = BQ_FILTER_RULE_TEMPLATE if self.config.use_v2_audit_metadata: @@ -388,12 +410,11 @@ def _compute_bigquery_lineage_via_gcp_logging( f"Error was {e}", ) - def _compute_bigquery_lineage_via_exported_bigquery_audit_metadata( - self, lineage_client_project_id: Optional[str] - ) -> None: + def _compute_bigquery_lineage_via_exported_bigquery_audit_metadata(self) -> None: + project_id = self.get_multiproject_project_id(run_on_compute=True) logger.info("Populating lineage info via exported GCP audit logs") try: - _client: BigQueryClient = BigQueryClient(project=lineage_client_project_id) + _client: BigQueryClient = BigQueryClient(project=project_id) exported_bigquery_audit_metadata: Iterable[ BigQueryAuditMetadata ] = self._get_exported_bigquery_audit_metadata(_client) @@ -410,27 +431,100 @@ def _compute_bigquery_lineage_via_exported_bigquery_audit_metadata( f"Error: {e}", ) - def _make_bigquery_client( - self, lineage_client_project_id: Optional[str] + def _make_gcp_logging_client( + self, project_id: Optional[str] ) -> List[GCPLoggingClient]: # See https://github.com/googleapis/google-cloud-python/issues/2674 for # why we disable gRPC here. client_options = self.config.extra_client_options.copy() client_options["_use_grpc"] = False - if lineage_client_project_id is not None: - return [ - GCPLoggingClient(**client_options, project=lineage_client_project_id) - ] + if project_id is not None: + return [GCPLoggingClient(**client_options, project=project_id)] else: return [GCPLoggingClient(**client_options)] - def _get_lineage_client_project_id(self) -> Optional[str]: - project_id: Optional[str] = ( - self.config.lineage_client_project_id - if self.config.lineage_client_project_id - else self.config.project_id + @staticmethod + def get_all_schema_tables_query(schema: str) -> str: + base_query = ( + f"SELECT " + f"table_id, " + f"size_bytes, " + f"last_modified_time, " + f"row_count, " + f"FROM {schema}.__TABLES__" + ) + return base_query + + def generate_profile_candidate_query( + self, threshold_time: Optional[datetime.datetime], schema: str + ) -> str: + row_condition = ( + f"row_count<{self.config.profiling.profile_table_row_limit} and " + if self.config.profiling.profile_table_row_limit + else "" + ) + size_condition = ( + f"ROUND(size_bytes/POW(10,9),2)<{self.config.profiling.profile_table_size_limit} and " + if self.config.profiling.profile_table_size_limit + else "" + ) + time_condition = ( + f"last_modified_time>={round(threshold_time.timestamp() * 1000)} and " + if threshold_time + else "" + ) + c = f"{row_condition}{size_condition}{time_condition}" + profile_clause = c if c == "" else f" WHERE {c}"[:-4] + if profile_clause == "": + return "" + query = f"{self.get_all_schema_tables_query(schema)}{profile_clause}" + logger.debug(f"Profiling via {query}") + return query + + def generate_profile_candidates( + self, + inspector: Inspector, + threshold_time: Optional[datetime.datetime], + schema: str, + ) -> Optional[List[str]]: + storage_project_id = self.get_multiproject_project_id(inspector) + exec_project_id = self.get_multiproject_project_id( + inspector, run_on_compute=True + ) + _client: BigQueryClient = BigQueryClient(project=exec_project_id) + + # if schema contains a bare dataset name, then add a project_id o/w dont modify it + full_schema_name = ( + f"{storage_project_id}.{schema}" if len(schema.split(".")) == 1 else schema ) - return project_id + # Reading all tables' metadata to report + all_tables = _client.query(self.get_all_schema_tables_query(full_schema_name)) + report_tables: List[str] = [ + "table_id, size_bytes, last_modified_time, row_count" + ] + for table_row in all_tables: + report_tables.append( + f"{table_row.table_id}, {table_row.size_bytes}, {table_row.last_modified_time}, {table_row.row_count}" + ) + report_key = f"{full_schema_name}" + self.report.table_metadata[report_key] = report_tables + + query = self.generate_profile_candidate_query(threshold_time, full_schema_name) + self.report.profile_table_selection_criteria[report_key] = ( + "no constraint" if query == "" else query.split(" WHERE")[1] + ) + if query == "": + return None + + query_job = _client.query(query) + _profile_candidates = [] + for row in query_job: + _profile_candidates.append(f"{full_schema_name}.{row.table_id}") + logger.debug( + f"Generated profiling candidates for {schema}: {_profile_candidates}" + ) + self.report.selected_profile_tables[report_key] = _profile_candidates + return _profile_candidates def _get_bigquery_log_entries( self, @@ -600,6 +694,7 @@ def _create_lineage_map(self, entries: Iterable[QueryEvent]) -> Dict[str, Set[st self.report.num_skipped_lineage_entries_missing_data = 0 self.report.num_skipped_lineage_entries_not_allowed = 0 self.report.num_skipped_lineage_entries_other = 0 + self.report.num_skipped_lineage_entries_sql_parser_failure = 0 for e in entries: self.report.num_total_lineage_entries += 1 if e.destinationTable is None or not ( @@ -608,7 +703,9 @@ def _create_lineage_map(self, entries: Iterable[QueryEvent]) -> Dict[str, Set[st self.report.num_skipped_lineage_entries_missing_data += 1 continue # Skip if schema/table pattern don't allow the destination table - destination_table_str = str(e.destinationTable.remove_extras()) + destination_table_str = str( + e.destinationTable.remove_extras(self.config.sharded_table_pattern) + ) destination_table_str_parts = destination_table_str.split("/") if not self.config.schema_pattern.allowed( destination_table_str_parts[3] @@ -617,13 +714,17 @@ def _create_lineage_map(self, entries: Iterable[QueryEvent]) -> Dict[str, Set[st continue has_table = False for ref_table in e.referencedTables: - ref_table_str = str(ref_table.remove_extras()) + ref_table_str = str( + ref_table.remove_extras(self.config.sharded_table_pattern) + ) if ref_table_str != destination_table_str: lineage_map[destination_table_str].add(ref_table_str) has_table = True has_view = False for ref_view in e.referencedViews: - ref_view_str = str(ref_view.remove_extras()) + ref_view_str = str( + ref_view.remove_extras(self.config.sharded_table_pattern) + ) if ref_view_str != destination_table_str: lineage_map[destination_table_str].add(ref_view_str) has_view = True @@ -631,10 +732,17 @@ def _create_lineage_map(self, entries: Iterable[QueryEvent]) -> Dict[str, Set[st # If there is a view being referenced then bigquery sends both the view as well as underlying table # in the references. There is no distinction between direct/base objects accessed. So doing sql parsing # to ensure we only use direct objects accessed for lineage - parser = BigQuerySQLParser(e.query) - referenced_objs = set( - map(lambda x: x.split(".")[-1], parser.get_tables()) - ) + try: + parser = BigQuerySQLParser(e.query) + referenced_objs = set( + map(lambda x: x.split(".")[-1], parser.get_tables()) + ) + except Exception as ex: + logger.warning( + f"Sql Parser failed on query: {e.query}. It will be skipped from lineage. The error was {ex}" + ) + self.report.num_skipped_lineage_entries_sql_parser_failure += 1 + continue curr_lineage_str = lineage_map[destination_table_str] new_lineage_str = set() for lineage_str in curr_lineage_str: @@ -646,15 +754,39 @@ def _create_lineage_map(self, entries: Iterable[QueryEvent]) -> Dict[str, Set[st self.report.num_skipped_lineage_entries_other += 1 return lineage_map + def is_table_partitioned( + self, database: Optional[str], schema: str, table: str + ) -> bool: + project_id: Optional[str] + if database: + project_id = database + else: + engine = self._get_engine(run_on_compute=True) + with engine.connect() as con: + inspector = inspect(con) + project_id = self.get_multiproject_project_id(inspector=inspector) + assert project_id + return f"{project_id}.{schema}.{table}" in self.partition_info + def get_latest_partition( self, schema: str, table: str ) -> Optional[BigQueryPartitionColumn]: - url = self.config.get_sql_alchemy_url() - engine = create_engine(url, **self.config.options) + logger.debug(f"get_latest_partition for {schema} and {table}") + engine = self._get_engine(run_on_compute=True) with engine.connect() as con: inspector = inspect(con) + project_id = self.get_multiproject_project_id(inspector=inspector) + assert project_id + if not self.is_table_partitioned( + database=project_id, schema=schema, table=table + ): + return None + project_id = self.get_multiproject_project_id(inspector=inspector) + assert project_id sql = BQ_GET_LATEST_PARTITION_TEMPLATE.format( - project_id=self.get_db_name(inspector), schema=schema, table=table + project_id=self.get_multiproject_project_id(inspector=inspector), + schema=schema, + table=table, ) result = con.execute(sql) # Bigquery only supports one partition column @@ -665,10 +797,10 @@ def get_latest_partition( return None def get_shard_from_table(self, table: str) -> Tuple[str, Optional[str]]: - match = re.search(SHARDED_TABLE_REGEX, table, re.IGNORECASE) + match = re.search(self.config.sharded_table_pattern, table, re.IGNORECASE) if match: - table_name = match.group(1) - shard = match.group(2) + table_name = match.group(2) + shard = match.group(3) return table_name, shard return table, None @@ -678,15 +810,19 @@ def is_latest_shard(self, project_id: str, schema: str, table: str) -> bool: table_name, shard = self.get_shard_from_table(table) if shard: logger.debug(f"{table_name} is sharded and shard id is: {shard}") - url = self.config.get_sql_alchemy_url() - engine = create_engine(url, **self.config.options) + engine = self._get_engine(run_on_compute=True) if f"{project_id}.{schema}.{table_name}" not in self.maximum_shard_ids: with engine.connect() as con: - sql = BQ_GET_LATEST_SHARD.format( - project_id=project_id, - schema=schema, - table=table_name, - ) + if table_name is not None: + sql = BQ_GET_LATEST_SHARD.format( + project_id=project_id, + schema=schema, + table=table_name, + ) + else: + sql = BQ_GET_LATEST_DATE_TABLE.format( + project_id=project_id, schema=schema, date_length=len(shard) + ) result = con.execute(sql) for row in result: @@ -703,10 +839,15 @@ def is_latest_shard(self, project_id: str, schema: str, table: str) -> bool: else: return True + def _get_engine(self, run_on_compute: bool = True) -> Engine: + url = self.config.get_sql_alchemy_url(run_on_compute=run_on_compute) + logger.debug(f"sql_alchemy_url={url}") + return create_engine(url, **self.config.options) + def add_information_for_schema(self, inspector: Inspector, schema: str) -> None: - url = self.config.get_sql_alchemy_url() - engine = create_engine(url, **self.config.options) - project_id = self.get_db_name(inspector) + engine = self._get_engine(run_on_compute=True) + project_id = self.get_multiproject_project_id(inspector=inspector) + assert project_id with engine.connect() as con: inspector = inspect(con) sql = f""" @@ -725,7 +866,8 @@ def get_extra_tags( self, inspector: Inspector, schema: str, table: str ) -> Dict[str, List[str]]: extra_tags: Dict[str, List[str]] = {} - project_id = self.get_db_name(inspector) + project_id = self.get_multiproject_project_id(inspector=inspector) + assert project_id partition_lookup_key = f"{project_id}.{schema}.{table}" if partition_lookup_key in self.partition_info: @@ -742,17 +884,29 @@ def generate_partition_profiler_query( partitioned table. See more about partitioned tables at https://cloud.google.com/bigquery/docs/partitioned-tables """ - + logger.debug( + f"generate partition profiler query for schema: {schema} and table {table}, partition_datetime: {partition_datetime}" + ) partition = self.get_latest_partition(schema, table) if partition: partition_where_clause: str logger.debug(f"{table} is partitioned and partition column is {partition}") - ( - partition_datetime, - upper_bound_partition_datetime, - ) = get_partition_range_from_partition_id( - partition.partition_id, partition_datetime - ) + try: + ( + partition_datetime, + upper_bound_partition_datetime, + ) = get_partition_range_from_partition_id( + partition.partition_id, partition_datetime + ) + except ValueError as e: + logger.error( + f"Unable to get partition range for partition id: {partition.partition_id} it failed with exception {e}" + ) + self.report.invalid_partition_ids[ + f"{schema}.{table}" + ] = partition.partition_id + return None, None + if partition.data_type in ("TIMESTAMP", "DATETIME"): partition_where_clause = "{column_name} BETWEEN '{partition_id}' AND '{upper_bound_partition_id}'".format( column_name=partition.column_name, @@ -790,14 +944,36 @@ def generate_partition_profiler_query( return shard, None return None, None + def get_profiler_instance(self, inspector: Inspector) -> "DatahubGEProfiler": + logger.debug("Getting profiler instance from bigquery") + engine = self._get_engine(run_on_compute=True) + with engine.connect() as conn: + inspector = inspect(conn) + + return DatahubGEProfiler( + conn=inspector.bind, + report=self.report, + config=self.config.profiling, + platform=self.platform, + ) + + def get_profile_args(self) -> Dict: + return {"temp_table_db": self.config.project_id} + def is_dataset_eligible_for_profiling( - self, dataset_name: str, sql_config: SQLAlchemyConfig + self, + dataset_name: str, + sql_config: SQLAlchemyConfig, + inspector: Inspector, + profile_candidates: Optional[List[str]], ) -> bool: """ Method overrides default profiling filter which checks profiling eligibility based on allow-deny pattern. This one also don't profile those sharded tables which are not the latest. """ - if not super().is_dataset_eligible_for_profiling(dataset_name, sql_config): + if not super().is_dataset_eligible_for_profiling( + dataset_name, sql_config, inspector, profile_candidates + ): return False (project_id, schema, table) = dataset_name.split(".") @@ -847,6 +1023,7 @@ def get_workunits(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: isinstance(wu, SqlWorkUnit) and isinstance(wu.metadata, MetadataChangeEvent) and isinstance(wu.metadata.proposedSnapshot, DatasetSnapshot) + and self.config.include_table_lineage ): lineage_mcp = self.get_lineage_mcp(wu.metadata.proposedSnapshot.urn) if lineage_mcp is not None: @@ -934,13 +1111,16 @@ def get_lineage_mcp( def prepare_profiler_args( self, + inspector: Inspector, schema: str, table: str, partition: Optional[str], custom_sql: Optional[str] = None, ) -> dict: + project_id = self._get_project_id(inspector=inspector) + assert project_id return dict( - schema=self.config.project_id, + schema=project_id, table=f"{schema}.{table}", partition=partition, custom_sql=custom_sql, @@ -960,7 +1140,7 @@ def normalise_dataset_name(self, dataset_name: str) -> str: BigQueryTableRef.from_spec_obj( {"projectId": project_id, "datasetId": schema, "tableId": table} ) - .remove_extras() + .remove_extras(self.config.sharded_table_pattern) .table ) return f"{project_id}.{schema}.{trimmed_table_name}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py index c4edbb2d347922..8fc6f69e1a5712 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py @@ -139,7 +139,8 @@ def get_sql_alchemy_url(self, database=None): PROPERTIES_COLUMNS = ( - "engine, partition_key, sorting_key, primary_key, sampling_key, storage_policy" + "engine, partition_key, sorting_key, primary_key, sampling_key, storage_policy, " + + "metadata_modification_time, total_rows, total_bytes, data_paths, metadata_path" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py index 1cde2c3dcc5997..40bb9b07bb6743 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py @@ -1,4 +1,5 @@ import json +import logging import re from typing import Any, Dict, List, Optional @@ -33,10 +34,62 @@ from datahub.utilities import config_clean from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column +logger = logging.getLogger(__name__) + register_custom_type(HiveDate, DateTypeClass) register_custom_type(HiveTimestamp, TimeTypeClass) register_custom_type(HiveDecimal, NumberTypeClass) +try: + + from databricks_dbapi.sqlalchemy_dialects.hive import DatabricksPyhiveDialect + from pyhive.sqlalchemy_hive import _type_map + from sqlalchemy import types, util + from sqlalchemy.engine import reflection + + @reflection.cache # type: ignore + def dbapi_get_columns_patched(self, connection, table_name, schema=None, **kw): + """Patches the get_columns method from dbapi (databricks_dbapi.sqlalchemy_dialects.base) to pass the native type through""" + rows = self._get_table_columns(connection, table_name, schema) + # Strip whitespace + rows = [[col.strip() if col else None for col in row] for row in rows] + # Filter out empty rows and comment + rows = [row for row in rows if row[0] and row[0] != "# col_name"] + result = [] + for (col_name, col_type, _comment) in rows: + # Handle both oss hive and Databricks' hive partition header, respectively + if col_name in ("# Partition Information", "# Partitioning"): + break + # Take out the more detailed type information + # e.g. 'map' -> 'map' + # 'decimal(10,1)' -> decimal + orig_col_type = col_type # keep a copy + col_type = re.search(r"^\w+", col_type).group(0) # type: ignore + try: + coltype = _type_map[col_type] + except KeyError: + util.warn( + "Did not recognize type '%s' of column '%s'" % (col_type, col_name) + ) + coltype = types.NullType # type: ignore + result.append( + { + "name": col_name, + "type": coltype, + "nullable": True, + "default": None, + "full_type": orig_col_type, # pass it through + "comment": _comment, + } + ) + return result + + DatabricksPyhiveDialect.get_columns = dbapi_get_columns_patched +except ModuleNotFoundError: + pass +except Exception as e: + logger.warning(f"Failed to patch method due to {e}") + class HiveConfig(BasicSQLAlchemyConfig): # defaults diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py index 9b05c794b477db..15d173f56b1b91 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql.py @@ -112,8 +112,8 @@ def _populate_table_descriptions(self, conn: Connection, db_name: str) -> None: SCHEMA_NAME(T.SCHEMA_ID) AS schema_name, T.NAME AS table_name, EP.VALUE AS table_description - FROM SYS.TABLES AS T - INNER JOIN SYS.EXTENDED_PROPERTIES AS EP + FROM sys.tables AS T + INNER JOIN sys.extended_properties AS EP ON EP.MAJOR_ID = T.[OBJECT_ID] AND EP.MINOR_ID = 0 AND EP.NAME = 'MS_Description' @@ -133,10 +133,10 @@ def _populate_column_descriptions(self, conn: Connection, db_name: str) -> None: T.NAME AS table_name, C.NAME AS column_name , EP.VALUE AS column_description - FROM SYS.TABLES AS T - INNER JOIN SYS.ALL_COLUMNS AS C + FROM sys.tables AS T + INNER JOIN sys.all_columns AS C ON C.OBJECT_ID = T.[OBJECT_ID] - INNER JOIN SYS.EXTENDED_PROPERTIES AS EP + INNER JOIN sys.extended_properties AS EP ON EP.MAJOR_ID = T.[OBJECT_ID] AND EP.MINOR_ID = C.COLUMN_ID AND EP.NAME = 'MS_Description' @@ -156,10 +156,10 @@ def create(cls, config_dict: Dict, ctx: PipelineContext) -> "SQLServerSource": # override to get table descriptions def get_table_properties( self, inspector: Inspector, schema: str, table: str - ) -> Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]]: + ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: description, properties, location_urn = super().get_table_properties( inspector, schema, table - ) # type:Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]] + ) # type:Tuple[Optional[str], Dict[str, str], Optional[str]] # Update description if available. db_name: str = self.get_db_name(inspector) description = self.table_descriptions.get( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py index 611f2f4b39a40a..e69589a15f3b6c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py @@ -2,7 +2,7 @@ from unittest.mock import patch # This import verifies that the dependencies are available. -import cx_Oracle # noqa: F401 +import cx_Oracle import pydantic from pydantic.fields import Field from sqlalchemy import event diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py index da4db1eb6b1ddf..c9edeb15559465 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/redshift.py @@ -7,7 +7,7 @@ # These imports verify that the dependencies are available. import psycopg2 # noqa: F401 -import pydantic # noqa: F401 +import pydantic import sqlalchemy import sqlalchemy_redshift # noqa: F401 from pydantic.fields import Field @@ -18,6 +18,7 @@ from sqllineage.runner import LineageRunner import datahub.emitter.mce_builder as builder +from datahub.configuration import ConfigModel from datahub.configuration.source_common import DatasetLineageProviderConfigBase from datahub.configuration.time_window_config import BaseTimeWindowConfig from datahub.emitter import mce_builder @@ -32,6 +33,8 @@ support_status, ) from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.aws.path_spec import PathSpec +from datahub.ingestion.source.aws.s3_util import strip_s3_prefix from datahub.ingestion.source.sql.postgres import PostgresConfig from datahub.ingestion.source.sql.sql_common import ( SQLAlchemySource, @@ -101,8 +104,31 @@ def __post_init__(self): self.dataset_lineage_type = DatasetLineageTypeClass.TRANSFORMED +class S3LineageProviderConfig(ConfigModel): + """ + Any source that produces s3 lineage from/to Datasets should inherit this class. + """ + + path_specs: List[PathSpec] = Field( + description="List of PathSpec. See below the details about PathSpec" + ) + + +class DatasetS3LineageProviderConfigBase(ConfigModel): + """ + Any source that produces s3 lineage from/to Datasets should inherit this class. + """ + + s3_lineage_config: Optional[S3LineageProviderConfig] = Field( + default=None, description="Common config for S3 lineage generation" + ) + + class RedshiftConfig( - PostgresConfig, BaseTimeWindowConfig, DatasetLineageProviderConfigBase + PostgresConfig, + BaseTimeWindowConfig, + DatasetLineageProviderConfigBase, + DatasetS3LineageProviderConfigBase, ): # Although Amazon Redshift is compatible with Postgres's wire format, # we actually want to use the sqlalchemy-redshift package and dialect @@ -672,6 +698,14 @@ def get_db_name(self, inspector: Inspector = None) -> str: db_name = db_alias return db_name + def _get_s3_path(self, path: str) -> str: + if self.config.s3_lineage_config: + for path_spec in self.config.s3_lineage_config.path_specs: + if path_spec.allowed(path): + table_name, table_path = path_spec.extract_table_name_and_path(path) + return table_path + return path + def _populate_lineage_map( self, query: str, lineage_type: LineageCollectorType ) -> None: @@ -747,6 +781,7 @@ def _populate_lineage_map( f"Only s3 source supported with copy. The source was: {path}.", ) continue + path = strip_s3_prefix(self._get_s3_path(path)) else: platform = LineageDatasetPlatform.REDSHIFT path = f'{db_name}.{db_row["source_schema"]}.{db_row["source_table"]}' diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py index 3d2d0c323363db..c3c0518ead788c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py @@ -1,6 +1,8 @@ import json import logging from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union import pydantic @@ -8,6 +10,7 @@ # This import verifies that the dependencies are available. import snowflake.sqlalchemy # noqa: F401 import sqlalchemy.engine +from snowflake import connector from snowflake.sqlalchemy import custom_types, snowdialect from sqlalchemy import create_engine, inspect from sqlalchemy.engine.reflection import Inspector @@ -24,6 +27,11 @@ platform_name, support_status, ) +from datahub.ingestion.api.source import ( + CapabilityReport, + TestableSource, + TestConnectionReport, +) from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.s3_util import make_s3_urn from datahub.ingestion.source.sql.sql_common import ( @@ -59,11 +67,13 @@ @support_status(SupportStatus.CERTIFIED) @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field") +@capability(SourceCapability.CONTAINERS, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration") @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion") -class SnowflakeSource(SQLAlchemySource): +class SnowflakeSource(SQLAlchemySource, TestableSource): def __init__(self, config: SnowflakeConfig, ctx: PipelineContext): super().__init__(config, ctx, "snowflake") self._lineage_map: Optional[Dict[str, List[Tuple[str, str, str]]]] = None @@ -71,12 +81,181 @@ def __init__(self, config: SnowflakeConfig, ctx: PipelineContext): self.report: SnowflakeReport = SnowflakeReport() self.config: SnowflakeConfig = config self.provision_role_in_progress: bool = False + self.profile_candidates: Dict[str, List[str]] = {} + + @staticmethod + def check_capabilities( + conn: connector.SnowflakeConnection, connection_conf: SnowflakeConfig + ) -> Dict[Union[SourceCapability, str], CapabilityReport]: + + # Currently only overall capabilities are reported. + # Resource level variations in capabilities are not considered. + + @dataclass + class SnowflakePrivilege: + privilege: str + object_name: str + object_type: str + + def query(query): + logger.info("Query : {}".format(query)) + resp = conn.cursor().execute(query) + return resp + + _report: Dict[Union[SourceCapability, str], CapabilityReport] = dict() + privileges: List[SnowflakePrivilege] = [] + capabilities: List[SourceCapability] = [c.capability for c in SnowflakeSource.get_capabilities() if c.capability not in (SourceCapability.PLATFORM_INSTANCE, SourceCapability.DOMAINS, SourceCapability.DELETION_DETECTION)] # type: ignore + + cur = query("select current_role()") + current_role = [row[0] for row in cur][0] + + cur = query("select current_secondary_roles()") + secondary_roles_str = json.loads([row[0] for row in cur][0])["roles"] + secondary_roles = ( + [] if secondary_roles_str == "" else secondary_roles_str.split(",") + ) + + roles = [current_role] + secondary_roles + + # PUBLIC role is automatically granted to every role + if "PUBLIC" not in roles: + roles.append("PUBLIC") + i = 0 + + while i < len(roles): + role = roles[i] + i = i + 1 + # for some roles, quoting is necessary. for example test-role + cur = query(f'show grants to role "{role}"') + for row in cur: + privilege = SnowflakePrivilege( + privilege=row[1], object_type=row[2], object_name=row[3] + ) + privileges.append(privilege) + + if privilege.object_type in ( + "DATABASE", + "SCHEMA", + ) and privilege.privilege in ("OWNERSHIP", "USAGE"): + _report[SourceCapability.CONTAINERS] = CapabilityReport( + capable=True + ) + elif privilege.object_type in ( + "TABLE", + "VIEW", + "MATERIALIZED_VIEW", + ): + _report[SourceCapability.SCHEMA_METADATA] = CapabilityReport( + capable=True + ) + _report[SourceCapability.DESCRIPTIONS] = CapabilityReport( + capable=True + ) + + if privilege.privilege in ("SELECT", "OWNERSHIP"): + _report[SourceCapability.DATA_PROFILING] = CapabilityReport( + capable=True + ) + + if privilege.object_name.startswith("SNOWFLAKE.ACCOUNT_USAGE."): + # if access to "snowflake" shared database, access to all account_usage views is automatically granted + # Finer access control is not yet supported for shares + # https://community.snowflake.com/s/article/Error-Granting-individual-privileges-on-imported-database-is-not-allowed-Use-GRANT-IMPORTED-PRIVILEGES-instead + _report[SourceCapability.LINEAGE_COARSE] = CapabilityReport( + capable=True + ) + # If all capabilities supported, no need to continue + if set(capabilities) == set(_report.keys()): + break + + # Due to this, entire role hierarchy is considered + if ( + privilege.object_type == "ROLE" + and privilege.privilege == "USAGE" + and privilege.object_name not in roles + ): + roles.append(privilege.object_name) + + cur = query("select current_warehouse()") + current_warehouse = [row[0] for row in cur][0] + + default_failure_messages = { + SourceCapability.SCHEMA_METADATA: "Either no tables exist or current role does not have permissions to access them", + SourceCapability.DESCRIPTIONS: "Either no tables exist or current role does not have permissions to access them", + SourceCapability.DATA_PROFILING: "Either no tables exist or current role does not have permissions to access them", + SourceCapability.CONTAINERS: "Current role does not have permissions to use any database", + SourceCapability.LINEAGE_COARSE: "Current role does not have permissions to snowflake account usage views", + } + + for c in capabilities: # type:ignore + + # These capabilities do not work without active warehouse + if current_warehouse is None and c in ( + SourceCapability.SCHEMA_METADATA, + SourceCapability.DESCRIPTIONS, + SourceCapability.DATA_PROFILING, + SourceCapability.LINEAGE_COARSE, + ): + failure_message = ( + f"Current role does not have permissions to use warehouse {connection_conf.warehouse}" + if connection_conf.warehouse is not None + else "No default warehouse set for user. Either set default warehouse for user or configure warehouse in recipe" + ) + _report[c] = CapabilityReport( + capable=False, + failure_reason=failure_message, + ) + + if c in _report.keys(): + continue + + # If some capabilities are missing, then mark them as not capable + _report[c] = CapabilityReport( + capable=False, + failure_reason=default_failure_messages[c], + ) + + return _report @classmethod def create(cls, config_dict, ctx): config = SnowflakeConfig.parse_obj(config_dict) return cls(config, ctx) + @staticmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + test_report = TestConnectionReport() + + try: + SnowflakeConfig.Config.extra = ( + pydantic.Extra.allow + ) # we are okay with extra fields during this stage + connection_conf = SnowflakeConfig.parse_obj(config_dict) + + connection: connector.SnowflakeConnection = connection_conf.get_connection() + assert connection + + test_report.basic_connectivity = CapabilityReport(capable=True) + + test_report.capability_report = SnowflakeSource.check_capabilities( + connection, connection_conf + ) + + except Exception as e: + logger.error(f"Failed to test connection due to {e}", exc_info=e) + if test_report.basic_connectivity is None: + test_report.basic_connectivity = CapabilityReport( + capable=False, failure_reason=f"{e}" + ) + else: + test_report.internal_failure = True + test_report.internal_failure_reason = f"{e}" + finally: + SnowflakeConfig.Config.extra = ( + pydantic.Extra.forbid + ) # set config flexibility back to strict + return test_report + def get_metadata_engine( self, database: Optional[str] = None ) -> sqlalchemy.engine.Engine: @@ -177,8 +356,12 @@ def get_inspectors(self) -> Iterable[Inspector]: else: self.report.report_dropped(db) - def get_identifier(self, *, schema: str, entity: str, **kwargs: Any) -> str: - regular = super().get_identifier(schema=schema, entity=entity, **kwargs) + def get_identifier( + self, *, schema: str, entity: str, inspector: Inspector, **kwargs: Any + ) -> str: + regular = super().get_identifier( + schema=schema, entity=entity, inspector=inspector, **kwargs + ) return f"{self.current_database.lower()}.{regular}" def _populate_view_upstream_lineage(self, engine: sqlalchemy.engine.Engine) -> None: @@ -748,6 +931,44 @@ def _is_dataset_allowed( return False return True + def generate_profile_candidates( + self, inspector: Inspector, threshold_time: Optional[datetime], schema: str + ) -> Optional[List[str]]: + if threshold_time is None: + return None + db_name = self.current_database + if self.profile_candidates.get(db_name) is not None: + # snowflake profile candidates are available at database level, + # no need to regenerate for every schema + return self.profile_candidates[db_name] + self.report.profile_if_updated_since = threshold_time + _profile_candidates = [] + logger.debug(f"Generating profiling candidates for db {db_name}") + db_rows = inspector.engine.execute( + text( + """ +select table_catalog, table_schema, table_name +from information_schema.tables +where last_altered >= to_timestamp_ltz({timestamp}, 3) and table_type= 'BASE TABLE' + """.format( + timestamp=round(threshold_time.timestamp() * 1000) + ) + ) + ) + + for db_row in db_rows: + _profile_candidates.append( + self.get_identifier( + schema=db_row.table_schema, + entity=db_row.table_name, + inspector=inspector, + ).lower() + ) + + self.report.profile_candidates[db_name] = _profile_candidates + self.profile_candidates[db_name] = _profile_candidates + return _profile_candidates + # Stateful Ingestion specific overrides # NOTE: There is no special state associated with this source yet than what is provided by sql_common. def get_platform_instance_id(self) -> str: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 1ea4270b07d981..692fc0ca1976f7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -92,6 +92,7 @@ ViewPropertiesClass, ) from datahub.telemetry import telemetry +from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport if TYPE_CHECKING: @@ -102,6 +103,8 @@ logger: logging.Logger = logging.getLogger(__name__) +MISSING_COLUMN_INFO = "missing column information" + def _platform_alchemy_uri_tester_gen( platform: str, opt_starts_with: Optional[str] = None @@ -239,23 +242,23 @@ class SQLAlchemyConfig(StatefulIngestionConfigBase): # them out afterwards via the table_pattern. schema_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), - description="regex patterns for schemas to filter in ingestion.", + description="Regex patterns for schemas to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'", ) table_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), - description="regex patterns for tables to filter in ingestion.", + description="Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", ) view_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), - description="regex patterns for views to filter in ingestion.", + description="Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", ) profile_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), - description="regex patterns for profiles to filter in ingestion, allowed by the `table_pattern`.", + description="Regex patterns to filter tables for profiling during ingestion. Allowed by the `table_pattern`.", ) domain: Dict[str, AllowDenyPattern] = Field( default=dict(), - description=' regex patterns for tables/schemas to descide domain_key domain key (domain_key can be any string like "sales".) There can be multiple domain key specified.', + description='Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like "Marketing".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.', ) include_views: Optional[bool] = Field( @@ -271,6 +274,17 @@ class SQLAlchemyConfig(StatefulIngestionConfigBase): # Custom Stateful Ingestion settings stateful_ingestion: Optional[SQLAlchemyStatefulIngestionConfig] = None + @pydantic.root_validator(pre=True) + def view_pattern_is_table_pattern_unless_specified( + cls, values: Dict[str, Any] + ) -> Dict[str, Any]: + view_pattern = values.get("view_pattern") + table_pattern = values.get("table_pattern") + if table_pattern and not view_pattern: + logger.info(f"Applying table_pattern {table_pattern} to view_pattern.") + values["view_pattern"] = table_pattern + return values + @pydantic.root_validator() def ensure_profiling_pattern_is_passed_to_profiling( cls, values: Dict[str, Any] @@ -287,7 +301,9 @@ def get_sql_alchemy_url(self): class BasicSQLAlchemyConfig(SQLAlchemyConfig): username: Optional[str] = Field(default=None, description="username") - password: Optional[pydantic.SecretStr] = Field(default=None, description="password") + password: Optional[pydantic.SecretStr] = Field( + default=None, exclude=True, description="password" + ) host_port: str = Field(description="host URL") database: Optional[str] = Field(default=None, description="database (catalog)") database_alias: Optional[str] = Field( @@ -493,6 +509,10 @@ def __init__(self, config: SQLAlchemyConfig, ctx: PipelineContext, platform: str for config_flag in profiling_flags_to_report }, ) + if self.config.domain: + self.domain_registry = DomainRegistry( + cached_domains=[k for k in self.config.domain], graph=self.ctx.graph + ) def warn(self, log: logging.Logger, key: str, reason: str) -> None: self.report.report_warning(key, reason) @@ -709,7 +729,7 @@ def get_workunits(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: profiler = None profile_requests: List["GEProfilerRequest"] = [] if sql_config.profiling.enabled: - profiler = self._get_profiler_instance(inspector) + profiler = self.get_profiler_instance(inspector) db_name = self.get_db_name(inspector) yield from self.gen_database_containers(db_name) @@ -753,7 +773,7 @@ def get_identifier( self, *, schema: str, entity: str, inspector: Inspector, **kwargs: Any ) -> str: # Many SQLAlchemy dialects have three-level hierarchies. This method, which - # subclasses can override, enables them to modify the identifers as needed. + # subclasses can override, enables them to modify the identifiers as needed. if hasattr(self.config, "get_identifier"): # This path is deprecated and will eventually be removed. return self.config.get_identifier(schema=schema, table=entity) # type: ignore @@ -805,7 +825,9 @@ def _gen_domain_urn(self, dataset_name: str) -> Optional[str]: for domain, pattern in self.config.domain.items(): if pattern.allowed(dataset_name): - domain_urn = make_domain_urn(domain) + domain_urn = make_domain_urn( + self.domain_registry.get_domain_urn(domain) + ) return domain_urn @@ -815,7 +837,7 @@ def _get_domain_wu( entity_urn: str, entity_type: str, sql_config: SQLAlchemyConfig, - ) -> Iterable[Union[MetadataWorkUnit]]: + ) -> Iterable[MetadataWorkUnit]: domain_urn = self._gen_domain_urn(dataset_name) if domain_urn: @@ -853,7 +875,6 @@ def loop_tables( # noqa: C901 continue self.report.report_entity_scanned(dataset_name, ent_type="table") - if not sql_config.table_pattern.allowed(dataset_name): self.report.report_dropped(dataset_name) continue @@ -912,8 +933,19 @@ def _process_table( description, properties, location_urn = self.get_table_properties( inspector, schema, table ) + + # Tablename might be different from the real table if we ran some normalisation ont it. + # Getting normalized table name from the dataset_name + # Table is the last item in the dataset name + normalised_table = table + splits = dataset_name.split(".") + if splits: + normalised_table = splits[-1] + if properties and normalised_table != table: + properties["original_table_name"] = table + dataset_properties = DatasetPropertiesClass( - name=table, + name=normalised_table, description=description, customProperties=properties, ) @@ -985,29 +1017,35 @@ def _process_table( def get_table_properties( self, inspector: Inspector, schema: str, table: str - ) -> Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]]: + ) -> Tuple[Optional[str], Dict[str, str], Optional[str]]: + description: Optional[str] = None + properties: Dict[str, str] = {} + + # The location cannot be fetched generically, but subclasses may override + # this method and provide a location. + location: Optional[str] = None + try: - location: Optional[str] = None - # SQLALchemy stubs are incomplete and missing this method. + # SQLAlchemy stubs are incomplete and missing this method. # PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223. table_info: dict = inspector.get_table_comment(table, schema) # type: ignore except NotImplementedError: - description: Optional[str] = None - properties: Dict[str, str] = {} + return description, properties, location except ProgrammingError as pe: # Snowflake needs schema names quoted when fetching table comments. logger.debug( f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and table {table}", pe, ) - description = None - properties = {} table_info: dict = inspector.get_table_comment(table, f'"{schema}"') # type: ignore - else: - description = table_info["text"] - # The "properties" field is a non-standard addition to SQLAlchemy's interface. - properties = table_info.get("properties", {}) + description = table_info.get("text") + if type(description) is tuple: + # Handling for value type tuple which is coming for dialect 'db2+ibm_db' + description = table_info["text"][0] + + # The "properties" field is a non-standard addition to SQLAlchemy's interface. + properties = table_info.get("properties", {}) return description, properties, location def get_dataplatform_instance_aspect( @@ -1040,7 +1078,7 @@ def _get_columns( try: columns = inspector.get_columns(table, schema) if len(columns) == 0: - self.report.report_warning(dataset_name, "missing column information") + self.report.report_warning(MISSING_COLUMN_INFO, dataset_name) except Exception as e: self.report.report_warning( dataset_name, @@ -1176,18 +1214,7 @@ def _process_view( columns, canonical_schema=schema_fields, ) - try: - # SQLALchemy stubs are incomplete and missing this method. - # PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223. - view_info: dict = inspector.get_table_comment(view, schema) # type: ignore - except NotImplementedError: - description: Optional[str] = None - properties: Dict[str, str] = {} - else: - description = view_info["text"] - - # The "properties" field is a non-standard addition to SQLAlchemy's interface. - properties = view_info.get("properties", {}) + description, properties, _ = self.get_table_properties(inspector, schema, view) try: view_definition = inspector.get_view_definition(view, schema) if view_definition is None: @@ -1285,7 +1312,7 @@ def add_table_to_schema_container( self.report.report_workunit(wu) yield wu - def _get_profiler_instance(self, inspector: Inspector) -> "DatahubGEProfiler": + def get_profiler_instance(self, inspector: Inspector) -> "DatahubGEProfiler": from datahub.ingestion.source.ge_data_profiler import DatahubGEProfiler return DatahubGEProfiler( @@ -1295,19 +1322,45 @@ def _get_profiler_instance(self, inspector: Inspector) -> "DatahubGEProfiler": platform=self.platform, ) + def get_profile_args(self) -> Dict: + """Passed down to GE profiler""" + return {} + # Override if needed def generate_partition_profiler_query( self, schema: str, table: str, partition_datetime: Optional[datetime.datetime] ) -> Tuple[Optional[str], Optional[str]]: return None, None + def is_table_partitioned( + self, database: Optional[str], schema: str, table: str + ) -> Optional[bool]: + return None + + # Override if needed + def generate_profile_candidates( + self, + inspector: Inspector, + threshold_time: Optional[datetime.datetime], + schema: str, + ) -> Optional[List[str]]: + raise NotImplementedError() + # Override if you want to do additional checks def is_dataset_eligible_for_profiling( - self, dataset_name: str, sql_config: SQLAlchemyConfig + self, + dataset_name: str, + sql_config: SQLAlchemyConfig, + inspector: Inspector, + profile_candidates: Optional[List[str]], ) -> bool: - return sql_config.table_pattern.allowed( - dataset_name - ) and sql_config.profile_pattern.allowed(dataset_name) + return ( + sql_config.table_pattern.allowed(dataset_name) + and sql_config.profile_pattern.allowed(dataset_name) + ) and ( + profile_candidates is None + or (profile_candidates is not None and dataset_name in profile_candidates) + ) def loop_profiler_requests( self, @@ -1318,6 +1371,25 @@ def loop_profiler_requests( from datahub.ingestion.source.ge_data_profiler import GEProfilerRequest tables_seen: Set[str] = set() + profile_candidates = None # Default value if profile candidates not available. + if ( + sql_config.profiling.profile_if_updated_since_days is not None + or sql_config.profiling.profile_table_size_limit is not None + or sql_config.profiling.profile_table_row_limit is None + ): + try: + threshold_time: Optional[datetime.datetime] = None + if sql_config.profiling.profile_if_updated_since_days is not None: + threshold_time = datetime.datetime.now( + datetime.timezone.utc + ) - datetime.timedelta( + sql_config.profiling.profile_if_updated_since_days + ) + profile_candidates = self.generate_profile_candidates( + inspector, threshold_time, schema + ) + except NotImplementedError: + logger.debug("Source does not support generating profile candidates.") for table in inspector.get_table_names(schema): schema, table = self.standardize_schema_table_names( @@ -1326,7 +1398,9 @@ def loop_profiler_requests( dataset_name = self.get_identifier( schema=schema, entity=table, inspector=inspector ) - if not self.is_dataset_eligible_for_profiling(dataset_name, sql_config): + if not self.is_dataset_eligible_for_profiling( + dataset_name, sql_config, inspector, profile_candidates + ): if self.config.profiling.report_dropped_profiles: self.report.report_dropped(f"profile of {dataset_name}") continue @@ -1339,10 +1413,26 @@ def loop_profiler_requests( logger.debug(f"{dataset_name} has already been seen, skipping...") continue + missing_column_info_warn = self.report.warnings.get(MISSING_COLUMN_INFO) + if ( + missing_column_info_warn is not None + and dataset_name in missing_column_info_warn + ): + continue + (partition, custom_sql) = self.generate_partition_profiler_query( schema, table, self.config.profiling.partition_datetime ) + if partition is None and self.is_table_partitioned( + database=None, schema=schema, table=table + ): + self.report.report_warning( + "profile skipped as partitioned table is empty or partition id was invalid", + dataset_name, + ) + continue + if ( partition is not None and not self.config.profiling.partition_profiling_enabled @@ -1353,9 +1443,13 @@ def loop_profiler_requests( continue self.report.report_entity_profiled(dataset_name) + logger.debug( + f"Preparing profiling request for {schema}, {table}, {partition}" + ) yield GEProfilerRequest( pretty_name=dataset_name, batch_kwargs=self.prepare_profiler_args( + inspector=inspector, schema=schema, table=table, partition=partition, @@ -1370,7 +1464,10 @@ def loop_profiler( platform: Optional[str] = None, ) -> Iterable[MetadataWorkUnit]: for request, profile in profiler.generate_profiles( - profile_requests, self.config.profiling.max_workers, platform=platform + profile_requests, + self.config.profiling.max_workers, + platform=platform, + profiler_args=self.get_profile_args(), ): if profile is None: continue @@ -1395,6 +1492,7 @@ def loop_profiler( def prepare_profiler_args( self, + inspector: Inspector, schema: str, table: str, partition: Optional[str], diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py index 2991a3a9311509..da5c5aa8d03f47 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py @@ -227,6 +227,16 @@ def resolve_postgres_modified_type(type_string: str) -> Any: return None +def resolve_trino_modified_type(type_string: str) -> Any: + # for cases like timestamp(3), decimal(10,0), row(...) + match = re.match(r"([a-zA-Z]+)\(.+\)", type_string) + if match: + modified_type_base: str = match.group(1) + return TRINO_SQL_TYPES_MAP[modified_type_base] + else: + return TRINO_SQL_TYPES_MAP[type_string] + + # see https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html SNOWFLAKE_TYPES_MAP: Dict[str, Any] = { "NUMBER": NumberType, @@ -308,3 +318,25 @@ def resolve_postgres_modified_type(type_string: str) -> Any: "struct": RecordType, "map": RecordType, } + +# https://trino.io/docs/current/language/types.html +# https://github.com/trinodb/trino-python-client/blob/master/trino/sqlalchemy/datatype.py#L75 +TRINO_SQL_TYPES_MAP = { + "boolean": BooleanType, + "tinyint": NumberType, + "smallint": NumberType, + "int": NumberType, + "integer": NumberType, + "bigint": NumberType, + "real": NumberType, + "double": NumberType, + "decimal": NumberType, + "varchar": StringType, + "char": StringType, + "varbinary": BytesType, + "json": RecordType, + "date": DateType, + "time": TimeType, + "timestamp": TimeType, + "row": RecordType, +} diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py index 80524cad6b7b57..ca174ff22bc3a1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py @@ -13,7 +13,7 @@ from sqlalchemy.engine.reflection import Inspector from sqlalchemy.sql import sqltypes from sqlalchemy.sql.type_api import TypeEngine -from trino.exceptions import TrinoQueryError # noqa +from trino.exceptions import TrinoQueryError from trino.sqlalchemy import datatype, error from trino.sqlalchemy.dialect import TrinoDialect @@ -71,8 +71,9 @@ def get_table_comment(self, connection, table_name: str, schema: str = None, **k # Generate properties dictionary. properties = {} - for col_name, col_value in row.items(): - properties[col_name] = col_value + if row: + for col_name, col_value in row.items(): + properties[col_name] = col_value return {"text": properties.get("comment", None), "properties": properties} except TrinoQueryError as e: diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py new file mode 100644 index 00000000000000..04c9abb81a165c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py @@ -0,0 +1,70 @@ +import logging +from typing import Callable, Dict, Iterable, List + +import pydantic + +from datahub.emitter.mce_builder import make_assertion_urn +from datahub.ingestion.source.state.checkpoint import CheckpointStateBase +from datahub.utilities.checkpoint_state_util import CheckpointStateUtil +from datahub.utilities.urns.urn import Urn + +logger = logging.getLogger(__name__) + + +class DbtCheckpointState(CheckpointStateBase): + """ + Class for representing the checkpoint state for DBT sources. + Stores all nodes and assertions being ingested and is used to remove any stale entities. + """ + + encoded_node_urns: List[str] = pydantic.Field(default_factory=list) + encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) + + @staticmethod + def _get_assertion_lightweight_repr(assertion_urn: str) -> str: + """Reduces the amount of text in the URNs for smaller state footprint.""" + urn = Urn.create_from_string(assertion_urn) + key = urn.get_entity_id_as_string() + assert key is not None + return key + + def add_assertion_urn(self, assertion_urn: str) -> None: + self.encoded_assertion_urns.append( + self._get_assertion_lightweight_repr(assertion_urn) + ) + + def get_assertion_urns_not_in( + self, checkpoint: "DbtCheckpointState" + ) -> Iterable[str]: + """ + Dbt assertion are mapped to DataHub assertion concept + """ + difference = CheckpointStateUtil.get_encoded_urns_not_in( + self.encoded_assertion_urns, checkpoint.encoded_assertion_urns + ) + for key in difference: + yield make_assertion_urn(key) + + def get_node_urns_not_in(self, checkpoint: "DbtCheckpointState") -> Iterable[str]: + """ + Dbt node are mapped to DataHub dataset concept + """ + yield from CheckpointStateUtil.get_dataset_urns_not_in( + self.encoded_node_urns, checkpoint.encoded_node_urns + ) + + def add_node_urn(self, node_urn: str) -> None: + self.encoded_node_urns.append( + CheckpointStateUtil.get_dataset_lightweight_repr(node_urn) + ) + + def set_checkpoint_urn(self, urn: str, entity_type: str) -> None: + supported_entities_add_handlers: Dict[str, Callable[[str], None]] = { + "dataset": self.add_node_urn, + "assertion": self.add_assertion_urn, + } + + if entity_type not in supported_entities_add_handlers: + logger.error(f"Can not save Unknown entity {entity_type} to checkpoint.") + + supported_entities_add_handlers[entity_type](urn) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py index 49ef4b69736410..8186232b30ee16 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py @@ -2,13 +2,9 @@ import pydantic -from datahub.emitter.mce_builder import ( - container_urn_to_key, - dataset_urn_to_key, - make_container_urn, - make_dataset_urn, -) +from datahub.emitter.mce_builder import container_urn_to_key, make_container_urn from datahub.ingestion.source.state.checkpoint import CheckpointStateBase +from datahub.utilities.checkpoint_state_util import CheckpointStateUtil class BaseSQLAlchemyCheckpointState(CheckpointStateBase): @@ -21,19 +17,12 @@ class BaseSQLAlchemyCheckpointState(CheckpointStateBase): encoded_table_urns: List[str] = pydantic.Field(default_factory=list) encoded_view_urns: List[str] = pydantic.Field(default_factory=list) encoded_container_urns: List[str] = pydantic.Field(default_factory=list) - - @staticmethod - def _get_separator() -> str: - # Unique small string not allowed in URNs. - return "||" + encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) @staticmethod def _get_lightweight_repr(dataset_urn: str) -> str: """Reduces the amount of text in the URNs for smaller state footprint.""" - SEP = BaseSQLAlchemyCheckpointState._get_separator() - key = dataset_urn_to_key(dataset_urn) - assert key is not None - return f"{key.platform}{SEP}{key.name}{SEP}{key.origin}" + return CheckpointStateUtil.get_dataset_lightweight_repr(dataset_urn) @staticmethod def _get_container_lightweight_repr(container_urn: str) -> str: @@ -42,36 +31,29 @@ def _get_container_lightweight_repr(container_urn: str) -> str: assert key is not None return f"{key.guid}" - @staticmethod - def _get_dataset_urns_not_in( - encoded_urns_1: List[str], encoded_urns_2: List[str] - ) -> Iterable[str]: - difference = set(encoded_urns_1) - set(encoded_urns_2) - for encoded_urn in difference: - platform, name, env = encoded_urn.split( - BaseSQLAlchemyCheckpointState._get_separator() - ) - yield make_dataset_urn(platform, name, env) - @staticmethod def _get_container_urns_not_in( encoded_urns_1: List[str], encoded_urns_2: List[str] ) -> Iterable[str]: - difference = set(encoded_urns_1) - set(encoded_urns_2) + difference = CheckpointStateUtil.get_encoded_urns_not_in( + encoded_urns_1, encoded_urns_2 + ) for guid in difference: yield make_container_urn(guid) def get_table_urns_not_in( self, checkpoint: "BaseSQLAlchemyCheckpointState" ) -> Iterable[str]: - yield from self._get_dataset_urns_not_in( + """Tables are mapped to DataHub dataset concept.""" + yield from CheckpointStateUtil.get_dataset_urns_not_in( self.encoded_table_urns, checkpoint.encoded_table_urns ) def get_view_urns_not_in( self, checkpoint: "BaseSQLAlchemyCheckpointState" ) -> Iterable[str]: - yield from self._get_dataset_urns_not_in( + """Views are mapped to DataHub dataset concept.""" + yield from CheckpointStateUtil.get_dataset_urns_not_in( self.encoded_view_urns, checkpoint.encoded_view_urns ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py index 066c1676eff918..9ae0aee12fabd3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py @@ -232,7 +232,7 @@ def get_last_checkpoint( ): return None - if JobId not in self.last_checkpoints: + if job_id not in self.last_checkpoints: self.last_checkpoints[job_id] = self._get_last_checkpoint( job_id, checkpoint_state_class ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 8e0dd6f90a086a..b8cdee3a29a384 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -4,7 +4,7 @@ import dateutil.parser as dp import requests -from pydantic.class_validators import validator +from pydantic.class_validators import root_validator, validator from pydantic.fields import Field from datahub.configuration.common import ConfigModel @@ -59,6 +59,10 @@ class SupersetConfig(ConfigModel): # See the Superset /security/login endpoint for details # https://superset.apache.org/docs/rest-api connect_uri: str = Field(default="localhost:8088", description="Superset host URL.") + display_uri: str = Field( + default=None, + description="optional URL to use in links (if `connect_uri` is only for ingestion)", + ) username: Optional[str] = Field(default=None, description="Superset username.") password: Optional[str] = Field(default=None, description="Superset password.") provider: str = Field(default="db", description="Superset provider.") @@ -72,10 +76,17 @@ class SupersetConfig(ConfigModel): description="Can be used to change mapping for database names in superset to what you have in datahub", ) - @validator("connect_uri") + @validator("connect_uri", "display_uri") def remove_trailing_slash(cls, v): return config_clean.remove_trailing_slashes(v) + @root_validator + def default_display_uri_to_connect_uri(cls, values): + base = values.get("display_uri") + if base is None: + values.set("display_uri", values.get("connect_uri")) + return values + def get_metric_name(metric): if not metric: @@ -208,7 +219,7 @@ def construct_dashboard_from_api_data(self, dashboard_data): created=AuditStamp(time=modified_ts, actor=modified_actor), lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) - dashboard_url = f"{self.config.connect_uri}{dashboard_data.get('url', '')}" + dashboard_url = f"{self.config.display_uri}{dashboard_data.get('url', '')}" chart_urns = [] raw_position_data = dashboard_data.get("position_json", "{}") @@ -280,7 +291,7 @@ def construct_chart_from_chart_data(self, chart_data): lastModified=AuditStamp(time=modified_ts, actor=modified_actor), ) chart_type = chart_type_from_viz_type.get(chart_data.get("viz_type", "")) - chart_url = f"{self.config.connect_uri}{chart_data.get('url', '')}" + chart_url = f"{self.config.display_uri}{chart_data.get('url', '')}" datasource_id = chart_data.get("datasource_id") datasource_urn = self.get_datasource_urn_from_id(datasource_id) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index fdb57ef4b543cc..e6e261a8fdd465 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import dateutil.parser as dp -from pydantic import validator +from pydantic import root_validator, validator from pydantic.fields import Field from tableauserverclient import ( PersonalAccessTokenAuth, @@ -132,10 +132,16 @@ class TableauConfig(ConfigModel): description="Ingest details for tables external to (not embedded in) tableau as entities.", ) - workbooks_page_size: int = Field( + workbooks_page_size: Optional[int] = Field( + default=None, + description="@deprecated(use page_size instead) Number of workbooks to query at a time using Tableau api.", + ) + + page_size: int = Field( default=10, - description="Number of workbooks to query at a time using Tableau api.", + description="Number of metadata objects (e.g. CustomSQLTable, PublishedDatasource, etc) to query at a time using Tableau api.", ) + env: str = Field( default=builder.DEFAULT_ENV, description="Environment to use in namespace when constructing URNs.", @@ -145,6 +151,17 @@ class TableauConfig(ConfigModel): def remove_trailing_slash(cls, v): return config_clean.remove_trailing_slashes(v) + @root_validator() + def show_warning_for_deprecated_config_field( + cls, values: Dict[str, Any] + ) -> Dict[str, Any]: + if values.get("workbooks_page_size") is not None: + logger.warn( + "Config workbooks_page_size is deprecated. Please use config page_size instead." + ) + + return values + class WorkbookKey(PlatformKey): workbook_id: str @@ -247,6 +264,9 @@ def get_connection_object( count: int = 0, current_count: int = 0, ) -> Tuple[dict, int, int]: + logger.debug( + f"Query {connection_type} to get {count} objects with offset {current_count}" + ) query_data = query_metadata( self.server, query, connection_type, count, current_count, query_filter ) @@ -267,7 +287,12 @@ def get_connection_object( has_next_page = connection_object.get("pageInfo", {}).get("hasNextPage", False) return connection_object, total_count, has_next_page - def emit_workbooks(self, workbooks_page_size: int) -> Iterable[MetadataWorkUnit]: + def emit_workbooks(self) -> Iterable[MetadataWorkUnit]: + count_on_query = ( + self.config.page_size + if self.config.workbooks_page_size is None + else self.config.workbooks_page_size + ) projects = ( f"projectNameWithin: {json.dumps(self.config.projects)}" @@ -282,8 +307,8 @@ def emit_workbooks(self, workbooks_page_size: int) -> Iterable[MetadataWorkUnit] current_count = 0 while has_next_page: count = ( - workbooks_page_size - if current_count + workbooks_page_size < total_count + count_on_query + if current_count + count_on_query < total_count else total_count - current_count ) ( @@ -305,7 +330,7 @@ def emit_workbooks(self, workbooks_page_size: int) -> Iterable[MetadataWorkUnit] yield from self.emit_sheets_as_charts(workbook) yield from self.emit_dashboards(workbook) yield from self.emit_embedded_datasource(workbook) - yield from self.emit_upstream_tables() + yield from self.emit_upstream_tables() def _track_custom_sql_ids(self, field: dict) -> None: # Tableau shows custom sql datasource as a table in ColumnField. @@ -355,18 +380,27 @@ def _create_upstream_table_lineage( return upstream_tables for table in datasource.get("upstreamTables", []): - # skip upstream tables when there is no column info when retrieving embedded datasource - # and when table name is None - # Schema details for these will be taken care in self.emit_custom_sql_ds() + # skip upstream tables when there is no column info when retrieving datasource + # Lineage and Schema details for these will be taken care in self.emit_custom_sql_datasources() if not is_custom_sql and not table.get("columns"): + logger.debug( + f"Skipping upstream table with id {table['id']}, no columns" + ) continue elif table["name"] is None: + logger.warning( + f"Skipping upstream table {table['id']} from lineage since its name is none" + ) continue schema = table.get("schema", "") table_name = table.get("name", "") full_name = table.get("fullName", "") - upstream_db = table.get("database", {}).get("name", "") + upstream_db = ( + table.get("database", {}).get("name", "") + if table.get("database") is not None + else "" + ) logger.debug( "Processing Table with Connection Type: {0} and id {1}".format( table.get("connectionType", ""), table.get("id", "") @@ -381,6 +415,9 @@ def _create_upstream_table_lineage( and table_name == full_name and schema in table_name ): + logger.debug( + f"Omitting schema for upstream table {table['id']}, schema included in table name" + ) schema = "" table_urn = make_table_urn( self.config.env, @@ -398,22 +435,20 @@ def _create_upstream_table_lineage( table_path = None if project and datasource.get("name"): - table_name = table.get("name") if table.get("name") else table["id"] + table_name = table.get("name") or table["id"] table_path = f"{project.replace('/', REPLACE_SLASH_CHAR)}/{datasource['name']}/{table_name}" self.upstream_tables[table_urn] = ( table.get("columns", []), table_path, - table.get("isEmbedded") if table.get("isEmbedded") else False, + table.get("isEmbedded") or False, ) return upstream_tables def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: count_on_query = len(self.custom_sql_ids_being_used) - custom_sql_filter = "idWithin: {}".format( - json.dumps(self.custom_sql_ids_being_used) - ) + custom_sql_filter = f"idWithin: {json.dumps(self.custom_sql_ids_being_used)}" custom_sql_connection, total_count, has_next_page = self.get_connection_object( custom_sql_graphql_query, "customSQLTablesConnection", custom_sql_filter ) @@ -491,7 +526,7 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: dataset_snapshot.aspects.append(schema_metadata) # Browse path - csql_name = csql.get("name") if csql.get("name") else csql_id + csql_name = csql.get("name") or csql_id if project and datasource_name: browse_paths = BrowsePathsClass( @@ -526,28 +561,34 @@ def emit_custom_sql_datasources(self) -> Iterable[MetadataWorkUnit]: def get_schema_metadata_for_custom_sql( self, columns: List[dict] ) -> Optional[SchemaMetadata]: + fields = [] schema_metadata = None for field in columns: # Datasource fields - fields = [] + + if field.get("name") is None: + logger.warning( + f"Skipping field {field['id']} from schema since its name is none" + ) + continue nativeDataType = field.get("remoteType", "UNKNOWN") TypeClass = FIELD_TYPE_MAPPING.get(nativeDataType, NullTypeClass) schema_field = SchemaField( - fieldPath=field.get("name", ""), + fieldPath=field["name"], type=SchemaFieldDataType(type=TypeClass()), nativeDataType=nativeDataType, description=field.get("description", ""), ) fields.append(schema_field) - schema_metadata = SchemaMetadata( - schemaName="test", - platform=f"urn:li:dataPlatform:{self.platform}", - version=0, - fields=fields, - hash="", - platformSchema=OtherSchema(rawSchema=""), - ) + schema_metadata = SchemaMetadata( + schemaName="test", + platform=f"urn:li:dataPlatform:{self.platform}", + version=0, + fields=fields, + hash="", + platformSchema=OtherSchema(rawSchema=""), + ) return schema_metadata def _create_lineage_from_csql_datasource( @@ -605,10 +646,14 @@ def _get_schema_metadata_for_datasource( self, datasource_fields: List[dict] ) -> Optional[SchemaMetadata]: fields = [] - schema_metadata = None for field in datasource_fields: # check datasource - custom sql relations from a field being referenced self._track_custom_sql_ids(field) + if field.get("name") is None: + logger.warning( + f"Skipping field {field['id']} from schema since its name is none" + ) + continue nativeDataType = field.get("dataType", "UNKNOWN") TypeClass = FIELD_TYPE_MAPPING.get(nativeDataType, NullTypeClass) @@ -632,8 +677,8 @@ def _get_schema_metadata_for_datasource( ) fields.append(schema_field) - if fields: - schema_metadata = SchemaMetadata( + return ( + SchemaMetadata( schemaName="test", platform=f"urn:li:dataPlatform:{self.platform}", version=0, @@ -641,8 +686,9 @@ def _get_schema_metadata_for_datasource( hash="", platformSchema=OtherSchema(rawSchema=""), ) - - return schema_metadata + if fields + else None + ) def get_metadata_change_event( self, snap_shot: Union["DatasetSnapshot", "DashboardSnapshot", "ChartSnapshot"] @@ -697,9 +743,7 @@ def emit_datasource( aspects=[], ) - datasource_name = ( - datasource.get("name") if datasource.get("name") else datasource_id - ) + datasource_name = datasource.get("name") or datasource_id if is_embedded_ds and workbook and workbook.get("name"): datasource_name = f"{workbook['name']}/{datasource_name}" # Browse path @@ -780,9 +824,7 @@ def emit_datasource( def emit_published_datasources(self) -> Iterable[MetadataWorkUnit]: count_on_query = len(self.datasource_ids_being_used) - datasource_filter = "idWithin: {}".format( - json.dumps(self.datasource_ids_being_used) - ) + datasource_filter = f"idWithin: {json.dumps(self.datasource_ids_being_used)}" ( published_datasource_conn, total_count, @@ -819,7 +861,7 @@ def emit_published_datasources(self) -> Iterable[MetadataWorkUnit]: def emit_upstream_tables(self) -> Iterable[MetadataWorkUnit]: for (table_urn, (columns, path, is_embedded)) in self.upstream_tables.items(): if not is_embedded and not self.config.ingest_tables_external: - logger.error( + logger.debug( f"Skipping external table {table_urn} as ingest_tables_external is set to False" ) continue @@ -840,6 +882,11 @@ def emit_upstream_tables(self) -> Iterable[MetadataWorkUnit]: if columns: fields = [] for field in columns: + if field.get("name") is None: + logger.warning( + f"Skipping field {field['id']} from schema since its name is none" + ) + continue nativeDataType = field.get("remoteType", "UNKNOWN") TypeClass = FIELD_TYPE_MAPPING.get(nativeDataType, NullTypeClass) @@ -881,7 +928,7 @@ def emit_sheets_as_charts(self, workbook: Dict) -> Iterable[MetadataWorkUnit]: aspects=[], ) - creator = workbook.get("owner", {}).get("username", "") + creator: Optional[str] = workbook["owner"].get("username") created_at = sheet.get("createdAt", datetime.now()) updated_at = sheet.get("updatedAt", datetime.now()) last_modified = self.get_last_modified(creator, created_at, updated_at) @@ -914,8 +961,6 @@ def emit_sheets_as_charts(self, workbook: Dict) -> Iterable[MetadataWorkUnit]: data_sources = self.get_sheetwise_upstream_datasources(sheet) for ds_id in data_sources: - if ds_id is None or not ds_id: - continue ds_urn = builder.make_dataset_urn(self.platform, ds_id, self.config.env) datasource_urn.append(ds_urn) if ds_id not in self.datasource_ids_being_used: @@ -933,7 +978,7 @@ def emit_sheets_as_charts(self, workbook: Dict) -> Iterable[MetadataWorkUnit]: chart_snapshot.aspects.append(chart_info) if workbook.get("projectName") and workbook.get("name"): - sheet_name = sheet.get("name") if sheet.get("name") else sheet["id"] + sheet_name = sheet.get("name") or sheet["id"] # Browse path browse_path = BrowsePathsClass( paths=[ @@ -1050,7 +1095,7 @@ def emit_dashboards(self, workbook: Dict) -> Iterable[MetadataWorkUnit]: dashboard_snapshot.aspects.append(dashboard_info_class) if workbook.get("projectName") and workbook.get("name"): - dashboard_name = title if title else dashboard["id"] + dashboard_name = title or dashboard["id"] # browse path browse_paths = BrowsePathsClass( paths=[ @@ -1104,13 +1149,13 @@ def _get_schema(self, schema_provided: str, database: str, fullName: str) -> str def _extract_schema_from_fullName(self, fullName: str) -> str: # fullName is observed to be in format [schemaName].[tableName] # OR simply tableName OR [tableName] - if fullName.startswith("[") and fullName.find("].[") >= 0: + if fullName.startswith("[") and "].[" in fullName: return fullName[1 : fullName.index("]")] return "" @lru_cache(maxsize=None) def get_last_modified( - self, creator: str, created_at: bytes, updated_at: bytes + self, creator: Optional[str], created_at: bytes, updated_at: bytes ) -> ChangeAuditStamps: last_modified = ChangeAuditStamps() if creator: @@ -1148,7 +1193,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: if self.server is None or not self.server.is_signed_in(): return try: - yield from self.emit_workbooks(self.config.workbooks_page_size) + yield from self.emit_workbooks() if self.datasource_ids_being_used: yield from self.emit_published_datasources() if self.custom_sql_ids_being_used: diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py index ddebc6a437ea4b..6296af05d14616 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py @@ -412,6 +412,9 @@ def make_table_urn( platform = "teradata" elif connection_type in ("sqlserver"): platform = "mssql" + elif connection_type in ("athena"): + platform = "athena" + upstream_db = "" else: platform = connection_type @@ -430,8 +433,7 @@ def make_table_urn( # if there are more than 3 tokens, just take the final 3 fully_qualified_table_name = ".".join(fully_qualified_table_name.split(".")[-3:]) - urn = builder.make_dataset_urn(platform, fully_qualified_table_name, env) - return urn + return builder.make_dataset_urn(platform, fully_qualified_table_name, env) def make_description_from_params(description, formula): @@ -448,10 +450,9 @@ def make_description_from_params(description, formula): def get_field_value_in_sheet(field, field_name): if field.get("__typename", "") == "DatasourceField": - field = field.get("remoteField") if field.get("remoteField") else {} + field = field.get("remoteField") or {} - field_value = field.get(field_name, "") - return field_value + return field.get(field_name, "") def get_unique_custom_sql(custom_sql_list: List[dict]) -> List[dict]: @@ -503,6 +504,4 @@ def query_metadata(server, main_query, connection_name, first, offset, qry_filte filter=qry_filter, main_query=main_query, ) - query_result = server.metadata.query(query) - - return query_result + return server.metadata.query(query) diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py index 7f14f5e5c2acf8..538e760f8ff6d9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py @@ -12,6 +12,7 @@ from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Union, cast import cachetools +from dateutil import parser from google.cloud.bigquery import Client as BigQueryClient from google.cloud.logging_v2.client import Client as GCPLoggingClient from more_itertools import partition @@ -66,6 +67,7 @@ PARTITIONED_TABLE_REGEX = re.compile( r"^(.+)[\$_](\d{4}|\d{6}|\d{8}|\d{10}|__PARTITIONS_SUMMARY__)$" ) +PARTITION_SUMMARY_REGEXP = re.compile(r"^(.+)\$__PARTITIONS_SUMMARY__$") # Handle table snapshots # See https://cloud.google.com/bigquery/docs/table-snapshots-intro. @@ -154,7 +156,6 @@ """.strip(), } - OPERATION_STATEMENT_TYPES = { "INSERT": OperationTypeClass.INSERT, "UPDATE": OperationTypeClass.UPDATE, @@ -166,6 +167,8 @@ "DROP_TABLE": OperationTypeClass.DROP, } +READ_STATEMENT_TYPES: List[str] = ["SELECT"] + def bigquery_audit_metadata_query_template( dataset: str, @@ -265,18 +268,42 @@ def is_temporary_table(self, prefix: str) -> bool: # Temporary tables will have a dataset that begins with an underscore. return self.dataset.startswith(prefix) - def remove_extras(self) -> "BigQueryTableRef": + @staticmethod + def remove_suffix(input_string, suffix): + if suffix and input_string.endswith(suffix): + return input_string[: -len(suffix)] + return input_string + + def remove_extras(self, sharded_table_regex: str) -> "BigQueryTableRef": # Handle partitioned and sharded tables. - matches = PARTITIONED_TABLE_REGEX.match(self.table) + table_name: Optional[str] = None + shortened_table_name = self.table + # if table name ends in _* or * then we strip it as that represents a query on a sharded table + shortened_table_name = self.remove_suffix(shortened_table_name, "_*") + shortened_table_name = self.remove_suffix(shortened_table_name, "*") + + matches = re.match(sharded_table_regex, shortened_table_name) if matches: - table_name = matches.group(1) + table_name = matches.group(2) + else: + matches = PARTITION_SUMMARY_REGEXP.match(shortened_table_name) + if matches: + table_name = matches.group(1) + if matches: + if not table_name: + logger.debug( + f"Using dataset id {self.dataset} as table name because table only contains date value {self.table}" + ) + table_name = self.dataset + logger.debug( - f"Found partitioned table {self.table}. Using {table_name} as the table name." + f"Found sharded table {self.table}. Using {table_name} as the table name." ) + return BigQueryTableRef(self.project, self.dataset, table_name) # Handle table snapshots. - matches = SNAPSHOT_TABLE_REGEX.match(self.table) + matches = SNAPSHOT_TABLE_REGEX.match(shortened_table_name) if matches: table_name = matches.group(1) logger.debug( @@ -286,14 +313,14 @@ def remove_extras(self) -> "BigQueryTableRef": # Handle exceptions invalid_chars_in_table_name: List[str] = [ - c for c in {"$", "@"} if c in self.table + c for c in {"$", "@"} if c in shortened_table_name ] if invalid_chars_in_table_name: raise ValueError( f"Cannot handle {self} - poorly formatted table name, contains {invalid_chars_in_table_name}" ) - return self + return BigQueryTableRef(self.project, self.dataset, shortened_table_name) def __str__(self) -> str: return f"projects/{self.project}/datasets/{self.dataset}/tables/{self.table}" @@ -311,8 +338,7 @@ def _table_ref_to_urn(ref: BigQueryTableRef, env: str) -> str: def _job_name_ref(project: str, jobId: str) -> Optional[str]: if project and jobId: return f"projects/{project}/jobs/{jobId}" - else: - return None + return None @dataclass @@ -334,7 +360,7 @@ class ReadEvent: # We really should use composition here since the query isn't actually # part of the read event, but this solution is just simpler. - query: Optional[str] = None # populated via join + # query: Optional["QueryEvent"] = None # populated via join @classmethod def get_missing_key_entry(cls, entry: AuditLogEntry) -> Optional[str]: @@ -434,12 +460,19 @@ class QueryEvent: timestamp: datetime actor_email: str query: str - statementType: Optional[str] = None + statementType: str + + job_name: Optional[str] = None destinationTable: Optional[BigQueryTableRef] = None referencedTables: List[BigQueryTableRef] = field(default_factory=list) referencedViews: List[BigQueryTableRef] = field(default_factory=list) - jobName: Optional[str] = None payload: Optional[Dict] = None + stats: Optional[Dict] = None + start_time: Optional[datetime] = None + end_time: Optional[datetime] = None + billed_bytes: Optional[int] = None + default_dataset: Optional[str] = None + numAffectedRows: Optional[int] = None @staticmethod def get_missing_key_entry(entry: AuditLogEntry) -> Optional[str]: @@ -462,10 +495,24 @@ def from_entry(cls, entry: AuditLogEntry) -> "QueryEvent": timestamp=entry.timestamp, actor_email=entry.payload["authenticationInfo"]["principalEmail"], query=job_query_conf["query"], - ) - # jobName - query_event.jobName = _job_name_ref( - job.get("jobName", {}).get("projectId"), job.get("jobName", {}).get("jobId") + job_name=_job_name_ref( + job.get("jobName", {}).get("projectId"), + job.get("jobName", {}).get("jobId"), + ), + default_dataset=job_query_conf["defaultDataset"] + if job_query_conf["defaultDataset"] + else None, + start_time=parser.parse(job["jobStatistics"]["startTime"]) + if job["jobStatistics"]["startTime"] + else None, + end_time=parser.parse(job["jobStatistics"]["endTime"]) + if job["jobStatistics"]["endTime"] + else None, + numAffectedRows=int(job["jobStatistics"]["queryOutputRowCount"]) + if "queryOutputRowCount" in job["jobStatistics"] + and job["jobStatistics"]["queryOutputRowCount"] + else None, + statementType=job_query_conf.get("statementType", "UNKNOWN"), ) # destinationTable raw_dest_table = job_query_conf.get("destinationTable") @@ -474,9 +521,11 @@ def from_entry(cls, entry: AuditLogEntry) -> "QueryEvent": raw_dest_table ) # statementType - query_event.statementType = job_query_conf.get("statementType") # referencedTables job_stats: Dict = job["jobStatistics"] + if job_stats.get("totalBilledBytes"): + query_event.billed_bytes = job_stats["totalBilledBytes"] + raw_ref_tables = job_stats.get("referencedTables") if raw_ref_tables: query_event.referencedTables = [ @@ -490,8 +539,7 @@ def from_entry(cls, entry: AuditLogEntry) -> "QueryEvent": ] # payload query_event.payload = entry.payload if DEBUG_INCLUDE_FULL_PAYLOADS else None - - if not query_event.jobName: + if not query_event.job_name: logger.debug( "jobName from query events is absent. " "Auditlog entry - {logEntry}".format(logEntry=entry) @@ -521,14 +569,30 @@ def from_exported_bigquery_audit_metadata( metadata: Dict = json.loads(row["metadata"]) job: Dict = metadata["jobChange"]["job"] query_config: Dict = job["jobConfig"]["queryConfig"] + query_stats: Dict = job["jobStats"]["queryStats"] + # basic query_event query_event = QueryEvent( timestamp=row["timestamp"], actor_email=payload["authenticationInfo"]["principalEmail"], query=query_config["query"], + job_name=job["jobName"], + default_dataset=query_config["defaultDataset"] + if query_config.get("defaultDataset") + else None, + start_time=parser.parse(job["jobStats"]["startTime"]) + if job["jobStats"]["startTime"] + else None, + end_time=parser.parse(job["jobStats"]["endTime"]) + if job["jobStats"]["endTime"] + else None, + numAffectedRows=int(query_stats["outputRowCount"]) + if query_stats.get("outputRowCount") + else None, + statementType=query_config.get("statementType", "UNKNOWN"), ) # jobName - query_event.jobName = job.get("jobName") + query_event.job_name = job.get("jobName") # destinationTable raw_dest_table = query_config.get("destinationTable") if raw_dest_table: @@ -536,7 +600,6 @@ def from_exported_bigquery_audit_metadata( raw_dest_table ) # referencedTables - query_stats: Dict = job["jobStats"]["queryStats"] raw_ref_tables = query_stats.get("referencedTables") if raw_ref_tables: query_event.referencedTables = [ @@ -548,17 +611,18 @@ def from_exported_bigquery_audit_metadata( query_event.referencedViews = [ BigQueryTableRef.from_string_name(spec) for spec in raw_ref_views ] - # statementType - query_event.statementType = query_config.get("statementType") # payload query_event.payload = payload if DEBUG_INCLUDE_FULL_PAYLOADS else None - if not query_event.jobName: + if not query_event.job_name: logger.debug( "jobName from query events is absent. " "BigQueryAuditMetadata entry - {logEntry}".format(logEntry=row) ) + if query_stats.get("totalBilledBytes"): + query_event.billed_bytes = int(query_stats["totalBilledBytes"]) + return query_event @classmethod @@ -567,13 +631,29 @@ def from_entry_v2(cls, row: BigQueryAuditMetadata) -> "QueryEvent": metadata: Dict = payload["metadata"] job: Dict = metadata["jobChange"]["job"] query_config: Dict = job["jobConfig"]["queryConfig"] + query_stats: Dict = job["jobStats"]["queryStats"] + # basic query_event query_event = QueryEvent( + job_name=job["jobName"], timestamp=row.timestamp, actor_email=payload["authenticationInfo"]["principalEmail"], query=query_config["query"], + default_dataset=query_config["defaultDataset"] + if "defaultDataset" in query_config and query_config["defaultDataset"] + else None, + start_time=parser.parse(job["jobStats"]["startTime"]) + if job["jobStats"]["startTime"] + else None, + end_time=parser.parse(job["jobStats"]["endTime"]) + if job["jobStats"]["endTime"] + else None, + numAffectedRows=int(query_stats["outputRowCount"]) + if "outputRowCount" in query_stats and query_stats["outputRowCount"] + else None, + statementType=query_config.get("statementType", "UNKNOWN"), ) - query_event.jobName = job.get("jobName") + query_event.job_name = job.get("jobName") # destinationTable raw_dest_table = query_config.get("destinationTable") if raw_dest_table: @@ -581,9 +661,7 @@ def from_entry_v2(cls, row: BigQueryAuditMetadata) -> "QueryEvent": raw_dest_table ) # statementType - query_event.statementType = query_config.get("statementType") # referencedTables - query_stats: Dict = job["jobStats"]["queryStats"] raw_ref_tables = query_stats.get("referencedTables") if raw_ref_tables: query_event.referencedTables = [ @@ -598,15 +676,24 @@ def from_entry_v2(cls, row: BigQueryAuditMetadata) -> "QueryEvent": # payload query_event.payload = payload if DEBUG_INCLUDE_FULL_PAYLOADS else None - if not query_event.jobName: + if not query_event.job_name: logger.debug( "jobName from query events is absent. " "BigQueryAuditMetadata entry - {logEntry}".format(logEntry=row) ) + if query_stats.get("totalBilledBytes"): + query_event.billed_bytes = int(query_stats["totalBilledBytes"]) + return query_event +@dataclass() +class AuditEvent: + read_event: Optional[ReadEvent] = None + query_event: Optional[QueryEvent] = None + + # We can't use close as it is not called if the ingestion is not successful def cleanup(config: BigQueryUsageConfig) -> None: if config._credentials_path is not None: @@ -696,18 +783,40 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: parsed_events: Iterable[Union[ReadEvent, QueryEvent]] = cast( Iterable[Union[ReadEvent, QueryEvent]], parsed_events_uncasted ) - last_updated_work_units: Iterable[MetadataWorkUnit] = cast( - Iterable[MetadataWorkUnit], last_updated_work_units_uncasted - ) - if self.config.include_operational_stats: - self.report.num_operational_stats_workunits_emitted = 0 - for wu in last_updated_work_units: - self.report.report_workunit(wu) - yield wu - self.report.num_operational_stats_workunits_emitted += 1 hydrated_read_events = self._join_events_by_job_id(parsed_events) - aggregated_info = self._aggregate_enriched_read_events(hydrated_read_events) + # storing it all in one big object. + aggregated_info: Dict[ + datetime, Dict[BigQueryTableRef, AggregatedDataset] + ] = collections.defaultdict(dict) + + # TODO: handle partitioned tables + + # TODO: perhaps we need to continuously prune this, rather than + num_aggregated: int = 0 + self.report.num_operational_stats_workunits_emitted = 0 + for event in hydrated_read_events: + if self.config.include_operational_stats: + operational_wu = self._create_operation_aspect_work_unit(event) + if operational_wu: + self.report.report_workunit(operational_wu) + yield operational_wu + self.report.num_operational_stats_workunits_emitted += 1 + if event.read_event: + aggregated_info = self._aggregate_enriched_read_events( + aggregated_info, event + ) + num_aggregated += 1 + logger.info(f"Total number of events aggregated = {num_aggregated}.") + bucket_level_stats: str = "\n\t" + "\n\t".join( + [ + f'bucket:{db.strftime("%m-%d-%Y:%H:%M:%S")}, size={len(ads)}' + for db, ads in aggregated_info.items() + ] + ) + logger.debug( + f"Number of buckets created = {len(aggregated_info)}. Per-bucket details:{bucket_level_stats}" + ) self.report.num_usage_workunits_emitted = 0 for time_bucket in aggregated_info.values(): @@ -837,51 +946,7 @@ def _get_bigquery_log_entries_via_gcp_logging( if self.config.use_v2_audit_metadata: audit_templates = BQ_AUDIT_V2 - # We adjust the filter values a bit, since we need to make sure that the join - # between query events and read events is complete. For example, this helps us - # handle the case where the read happens within our time range but the query - # completion event is delayed and happens after the configured end time. - - # Can safely access the first index of the allow list as it by default contains ".*" - use_allow_filter = self.config.table_pattern and ( - len(self.config.table_pattern.allow) > 1 - or self.config.table_pattern.allow[0] != ".*" - ) - use_deny_filter = self.config.table_pattern and self.config.table_pattern.deny - allow_regex = ( - audit_templates["BQ_FILTER_REGEX_ALLOW_TEMPLATE"].format( - allow_pattern=self.config.get_allow_pattern_string() - ) - if use_allow_filter - else "" - ) - deny_regex = ( - audit_templates["BQ_FILTER_REGEX_DENY_TEMPLATE"].format( - deny_pattern=self.config.get_deny_pattern_string(), - logical_operator="AND" if use_allow_filter else "", - ) - if use_deny_filter - else ("" if use_allow_filter else "FALSE") - ) - - logger.debug( - f"use_allow_filter={use_allow_filter}, use_deny_filter={use_deny_filter}, " - f"allow_regex={allow_regex}, deny_regex={deny_regex}" - ) - start_time = (self.config.start_time - self.config.max_query_duration).strftime( - BQ_DATETIME_FORMAT - ) - self.report.log_entry_start_time = start_time - end_time = (self.config.end_time + self.config.max_query_duration).strftime( - BQ_DATETIME_FORMAT - ) - self.report.log_entry_end_time = end_time - filter = audit_templates["BQ_FILTER_RULE_TEMPLATE"].format( - start_time=start_time, - end_time=end_time, - allow_regex=allow_regex, - deny_regex=deny_regex, - ) + filter = self._generate_filter(audit_templates) logger.debug(filter) list_entry_generators_across_clients: List[ @@ -923,58 +988,206 @@ def _get_bigquery_log_entries_via_gcp_logging( yield entry logger.info(f"Finished loading {i} log entries from GCP Logging") + def _generate_filter(self, audit_templates: Dict[str, str]) -> str: + # We adjust the filter values a bit, since we need to make sure that the join + # between query events and read events is complete. For example, this helps us + # handle the case where the read happens within our time range but the query + # completion event is delayed and happens after the configured end time. + # Can safely access the first index of the allow list as it by default contains ".*" + use_allow_filter = self.config.table_pattern and ( + len(self.config.table_pattern.allow) > 1 + or self.config.table_pattern.allow[0] != ".*" + ) + use_deny_filter = self.config.table_pattern and self.config.table_pattern.deny + allow_regex = ( + audit_templates["BQ_FILTER_REGEX_ALLOW_TEMPLATE"].format( + allow_pattern=self.config.get_allow_pattern_string() + ) + if use_allow_filter + else "" + ) + base_deny_pattern: str = "__TABLES_SUMMARY__|INFORMATION_SCHEMA" + deny_regex = audit_templates["BQ_FILTER_REGEX_DENY_TEMPLATE"].format( + deny_pattern=base_deny_pattern + "|" + self.config.get_deny_pattern_string() + if self.config.get_deny_pattern_string() + else base_deny_pattern, + logical_operator="AND" if use_allow_filter else "", + ) + logger.debug( + f"use_allow_filter={use_allow_filter}, use_deny_filter={use_deny_filter}, " + f"allow_regex={allow_regex}, deny_regex={deny_regex}" + ) + start_time = (self.config.start_time - self.config.max_query_duration).strftime( + BQ_DATETIME_FORMAT + ) + self.report.log_entry_start_time = start_time + end_time = (self.config.end_time + self.config.max_query_duration).strftime( + BQ_DATETIME_FORMAT + ) + self.report.log_entry_end_time = end_time + filter = audit_templates["BQ_FILTER_RULE_TEMPLATE"].format( + start_time=start_time, + end_time=end_time, + allow_regex=allow_regex, + deny_regex=deny_regex, + ) + return filter + def _create_operation_aspect_work_unit( - self, event: QueryEvent + self, event: AuditEvent ) -> Optional[MetadataWorkUnit]: - if event.statementType in OPERATION_STATEMENT_TYPES and event.destinationTable: - destination_table: BigQueryTableRef - try: - destination_table = event.destinationTable.remove_extras() - except Exception as e: - self.report.report_warning( - str(event.destinationTable), - f"Failed to clean up destination table, {e}", - ) - return None - reported_time: int = int(time.time() * 1000) - last_updated_timestamp: int = int(event.timestamp.timestamp() * 1000) - affected_datasets = [] - if event.referencedTables: - for table in event.referencedTables: - try: - affected_datasets.append( - _table_ref_to_urn( - table.remove_extras(), - self.config.env, - ) - ) - except Exception as e: - self.report.report_warning( - str(table), - f"Failed to clean up table, {e}", - ) - operation_aspect = OperationClass( - timestampMillis=reported_time, - lastUpdatedTimestamp=last_updated_timestamp, - actor=builder.make_user_urn(event.actor_email.split("@")[0]), - operationType=OPERATION_STATEMENT_TYPES[event.statementType], - affectedDatasets=affected_datasets, + + if not event.read_event and not event.query_event: + return None + + destination_table: BigQueryTableRef + if ( + not event.read_event + and event.query_event + and event.query_event.destinationTable + ): + destination_table = event.query_event.destinationTable.remove_extras( + self.config.sharded_table_pattern ) - mcp = MetadataChangeProposalWrapper( - entityType="dataset", - aspectName="operation", - changeType=ChangeTypeClass.UPSERT, - entityUrn=_table_ref_to_urn( - destination_table, - env=self.config.env, - ), - aspect=operation_aspect, + elif event.read_event: + destination_table = event.read_event.resource.remove_extras( + self.config.sharded_table_pattern ) - return MetadataWorkUnit( - id=f"{event.timestamp.isoformat()}-operation-aspect-{destination_table}", - mcp=mcp, + else: + # TODO: CREATE_SCHEMA operation ends up here, maybe we should capture that as well + # but it is tricky as we only get the query so it can't be tied to anything + # - SCRIPT statement type ends up here as well + logger.warning(f"Unable to find destination table in event {event}") + return None + + if not self._is_table_allowed(destination_table): + return None + + statement_type: str + custom_type: Optional[str] = None + last_updated_timestamp: int + actor_email: str + # If we don't have Query object that means this is a queryless read operation or a read operation which was not executed as JOB + # https://cloud.google.com/bigquery/docs/reference/auditlogs/rest/Shared.Types/BigQueryAuditMetadata.TableDataRead.Reason/ + if not event.query_event and event.read_event: + statement_type = OperationTypeClass.CUSTOM + custom_type = "CUSTOM_READ" + last_updated_timestamp = int(event.read_event.timestamp.timestamp() * 1000) + actor_email = event.read_event.actor_email + elif event.query_event: + # If AuditEvent only have queryEvent that means it is the target of the Insert Operation + if ( + event.query_event.statementType in OPERATION_STATEMENT_TYPES + and not event.read_event + ): + statement_type = OPERATION_STATEMENT_TYPES[ + event.query_event.statementType + ] + # We don't have SELECT in OPERATION_STATEMENT_TYPES , so those queries will end up here + # and this part should capture those operation types as well which we don't have in our mapping + else: + statement_type = OperationTypeClass.CUSTOM + custom_type = event.query_event.statementType + + self.report.operation_types_stat[event.query_event.statementType] = ( + self.report.operation_types_stat.get(event.query_event.statementType, 0) + + 1 ) - return None + last_updated_timestamp = int(event.query_event.timestamp.timestamp() * 1000) + actor_email = event.query_event.actor_email + else: + return None + + if not self.config.include_read_operational_stats and ( + statement_type not in OPERATION_STATEMENT_TYPES.values() + ): + return None + + reported_time: int = int(time.time() * 1000) + affected_datasets = [] + if event.query_event and event.query_event.referencedTables: + for table in event.query_event.referencedTables: + try: + affected_datasets.append( + _table_ref_to_urn( + table.remove_extras(self.config.sharded_table_pattern), + self.config.env, + ) + ) + except Exception as e: + self.report.report_warning( + str(table), + f"Failed to clean up table, {e}", + ) + + operation_aspect = OperationClass( + timestampMillis=reported_time, + lastUpdatedTimestamp=last_updated_timestamp, + actor=builder.make_user_urn(actor_email.split("@")[0]), + operationType=statement_type, + customOperationType=custom_type if custom_type else None, + affectedDatasets=affected_datasets, + ) + + if self.config.include_read_operational_stats: + operation_aspect.customProperties = ( + self._create_operational_custom_properties(event) + ) + if event.query_event and event.query_event.numAffectedRows: + operation_aspect.numAffectedRows = event.query_event.numAffectedRows + + mcp = MetadataChangeProposalWrapper( + entityType="dataset", + aspectName="operation", + changeType=ChangeTypeClass.UPSERT, + entityUrn=_table_ref_to_urn( + destination_table, + env=self.config.env, + ), + aspect=operation_aspect, + ) + return MetadataWorkUnit( + id=f"{datetime.fromtimestamp(last_updated_timestamp / 1000).isoformat()}-operation-aspect-{destination_table}", + mcp=mcp, + ) + + def _create_operational_custom_properties( + self, event: AuditEvent + ) -> Dict[str, str]: + custom_properties: Dict[str, str] = {} + # This only needs for backward compatibility reason. To make sure we generate the same operational metadata than before + if self.config.include_read_operational_stats: + if event.query_event: + if event.query_event.end_time and event.query_event.start_time: + custom_properties["millisecondsTaken"] = str( + int(event.query_event.end_time.timestamp() * 1000) + - int(event.query_event.start_time.timestamp() * 1000) + ) + + if event.query_event.job_name: + custom_properties["sessionId"] = event.query_event.job_name + + custom_properties["text"] = event.query_event.query + + if event.query_event.billed_bytes: + custom_properties["bytesProcessed"] = str( + event.query_event.billed_bytes + ) + + if event.query_event.default_dataset: + custom_properties[ + "defaultDatabase" + ] = event.query_event.default_dataset + if event.read_event: + if event.read_event.readReason: + custom_properties["readReason"] = event.read_event.readReason + + if event.read_event.fieldsRead: + custom_properties["fieldsRead"] = ",".join( + event.read_event.fieldsRead + ) + + return custom_properties def _parse_bigquery_log_entries( self, entries: Iterable[Union[AuditLogEntry, BigQueryAuditMetadata]] @@ -992,30 +1205,23 @@ def _parse_bigquery_log_entries( if not self._is_table_allowed(event.resource): self.report.num_filtered_read_events += 1 continue + + if event.readReason: + self.report.read_reasons_stat[event.readReason] = ( + self.report.read_reasons_stat.get(event.readReason, 0) + 1 + ) self.report.num_read_events += 1 missing_query_entry = QueryEvent.get_missing_key_entry(entry) if event is None and missing_query_entry is None: event = QueryEvent.from_entry(entry) - if not self._is_table_allowed(event.destinationTable): - self.report.num_filtered_query_events += 1 - continue self.report.num_query_events += 1 - wu = self._create_operation_aspect_work_unit(event) - if wu: - yield wu missing_query_entry_v2 = QueryEvent.get_missing_key_entry_v2(entry) if event is None and missing_query_entry_v2 is None: event = QueryEvent.from_entry_v2(entry) - if not self._is_table_allowed(event.destinationTable): - self.report.num_filtered_query_events += 1 - continue self.report.num_query_events += 1 - wu = self._create_operation_aspect_work_unit(event) - if wu: - yield wu if event is None: self.error( @@ -1024,7 +1230,6 @@ def _parse_bigquery_log_entries( f"Unable to parse {type(entry)} missing read {missing_query_entry}, missing query {missing_query_entry} missing v2 {missing_query_entry_v2} for {entry}", ) else: - logger.debug(f"Yielding {event} from log entries") yield event logger.info( @@ -1043,9 +1248,6 @@ def _parse_exported_bigquery_audit_metadata( ) if missing_query_event_exported_audit is None: event = QueryEvent.from_exported_bigquery_audit_metadata(audit_metadata) - wu = self._create_operation_aspect_work_unit(event) - if wu: - yield wu missing_read_event_exported_audit = ( ReadEvent.get_missing_key_exported_bigquery_audit_metadata( @@ -1072,7 +1274,7 @@ def error(self, log: logging.Logger, key: str, reason: str) -> Any: def _join_events_by_job_id( self, events: Iterable[Union[ReadEvent, QueryEvent]] - ) -> Iterable[ReadEvent]: + ) -> Iterable[AuditEvent]: # If caching eviction is enabled, we only store the most recently used query events, # which are used when resolving job information within the read events. query_jobs: MutableMapping[str, QueryEvent] @@ -1083,13 +1285,18 @@ def _join_events_by_job_id( def event_processor( events: Iterable[Union[ReadEvent, QueryEvent]] - ) -> Iterable[ReadEvent]: + ) -> Iterable[AuditEvent]: for event in events: if isinstance(event, QueryEvent): - if event.jobName: - query_jobs[event.jobName] = event + if event.job_name: + query_jobs[event.job_name] = event + # For Insert operations we yield the query event as it is possible + # there won't be any read event. + if event.statementType not in READ_STATEMENT_TYPES: + yield AuditEvent(query_event=event) + # If destination table exists we yield the query event as it is insert operation else: - yield event + yield AuditEvent(read_event=event) # TRICKY: To account for the possibility that the query event arrives after # the read event in the audit logs, we wait for at least `query_log_delay` @@ -1103,77 +1310,79 @@ def event_processor( num_joined: int = 0 for event in delayed_read_events: + # If event_processor yields a query event which is an insert operation + # then we should just yield it. + if event.query_event and not event.read_event: + yield event + continue if ( - event.timestamp < self.config.start_time - or event.timestamp >= self.config.end_time - or not self._is_table_allowed(event.resource) + event.read_event is None + or event.read_event.timestamp < self.config.start_time + or event.read_event.timestamp >= self.config.end_time + or not self._is_table_allowed(event.read_event.resource) ): continue - if event.jobName: - if event.jobName in query_jobs: + # There are some read event which does not have jobName because it was read in a different way + # Like https://cloud.google.com/logging/docs/reference/audit/bigquery/rest/Shared.Types/AuditData#tabledatalistrequest + # There are various reason to read a table + # https://cloud.google.com/bigquery/docs/reference/auditlogs/rest/Shared.Types/BigQueryAuditMetadata.TableDataRead.Reason + if event.read_event.jobName: + if event.read_event.jobName in query_jobs: # Join the query log event into the table read log event. num_joined += 1 - event.query = query_jobs[event.jobName].query - - # TODO also join into the query itself for column references + event.query_event = query_jobs[event.read_event.jobName] else: self.report.report_warning( - str(event.resource), - f"Failed to match table read event {event.jobName} with job; try increasing `query_log_delay` or `max_query_duration`", + str(event.read_event.resource), + f"Failed to match table read event {event.read_event.jobName} with job; try increasing `query_log_delay` or `max_query_duration`", ) yield event - logger.info(f"Number of read events joined with query events: {num_joined}") def _aggregate_enriched_read_events( - self, events: Iterable[ReadEvent] + self, + datasets: Dict[datetime, Dict[BigQueryTableRef, AggregatedDataset]], + event: AuditEvent, ) -> Dict[datetime, Dict[BigQueryTableRef, AggregatedDataset]]: - # TODO: handle partitioned tables + if not event.read_event: + return datasets - # TODO: perhaps we need to continuously prune this, rather than - # storing it all in one big object. - datasets: Dict[ - datetime, Dict[BigQueryTableRef, AggregatedDataset] - ] = collections.defaultdict(dict) - - num_aggregated: int = 0 - for event in events: - floored_ts = get_time_bucket(event.timestamp, self.config.bucket_duration) - resource: Optional[BigQueryTableRef] = None - try: - resource = event.resource.remove_extras() - except Exception as e: - self.report.report_warning( - str(event.resource), f"Failed to clean up resource, {e}" - ) - logger.warning(f"Failed to process event {str(event.resource)}", e) - continue - - if resource.is_temporary_table(self.config.temp_table_dataset_prefix): - logger.debug(f"Dropping temporary table {resource}") - self.report.report_dropped(str(resource)) - continue - - agg_bucket = datasets[floored_ts].setdefault( - resource, - AggregatedDataset( - bucket_start_time=floored_ts, - resource=resource, - user_email_pattern=self.config.user_email_pattern, - ), + floored_ts = get_time_bucket( + event.read_event.timestamp, self.config.bucket_duration + ) + resource: Optional[BigQueryTableRef] = None + try: + resource = event.read_event.resource.remove_extras( + self.config.sharded_table_pattern ) - agg_bucket.add_read_entry(event.actor_email, event.query, event.fieldsRead) - num_aggregated += 1 - logger.info(f"Total number of events aggregated = {num_aggregated}.") - bucket_level_stats: str = "\n\t" + "\n\t".join( - [ - f'bucket:{db.strftime("%m-%d-%Y:%H:%M:%S")}, size={len(ads)}' - for db, ads in datasets.items() - ] + except Exception as e: + self.report.report_warning( + str(event.read_event.resource), f"Failed to clean up resource, {e}" + ) + logger.warning( + f"Failed to process event {str(event.read_event.resource)}", e + ) + return datasets + + if resource.is_temporary_table(self.config.temp_table_dataset_prefix): + logger.debug(f"Dropping temporary table {resource}") + self.report.report_dropped(str(resource)) + return datasets + + agg_bucket = datasets[floored_ts].setdefault( + resource, + AggregatedDataset( + bucket_start_time=floored_ts, + resource=resource, + user_email_pattern=self.config.user_email_pattern, + ), ) - logger.debug( - f"Number of buckets created = {len(datasets)}. Per-bucket details:{bucket_level_stats}" + + agg_bucket.add_read_entry( + event.read_event.actor_email, + event.query_event.query if event.query_event else None, + event.read_event.fieldsRead, ) return datasets diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py b/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py index 283c8c358fdefb..0f63b5cdc7f54a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py @@ -148,6 +148,12 @@ class BaseUsageConfig(BaseTimeWindowConfig): include_operational_stats: bool = Field( default=True, description="Whether to display operational stats." ) + + include_read_operational_stats: bool = Field( + default=False, + description="Whether to report read operational stats. Experimental.", + ) + format_sql_queries: bool = Field( default=False, description="Whether to format sql queries" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py index ca6b6737aae5b7..b33b985e417144 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/bigquery.py @@ -1,6 +1,10 @@ +import re + import pydantic -from datahub.configuration.common import ConfigModel +from datahub.configuration.common import ConfigModel, ConfigurationError + +_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX: str = "((.+)[_$])?(\\d{4,10})$" class BigQueryBaseConfig(ConfigModel): @@ -11,3 +15,23 @@ class BigQueryBaseConfig(ConfigModel): default=60, description="Used to control number of API calls made per min. Only used when `rate_limit` is set to `True`.", ) + + temp_table_dataset_prefix: str = pydantic.Field( + default="_", + description="If you are creating temp tables in a dataset with a particular prefix you can use this config to set the prefix for the dataset. This is to support workflows from before bigquery's introduction of temp tables. By default we use `_` because of datasets that begin with an underscore are hidden by default https://cloud.google.com/bigquery/docs/datasets#dataset-naming.", + ) + + sharded_table_pattern: str = pydantic.Field( + default=_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX, + description="The regex pattern to match sharded tables and group as one table. This is a very low level config parameter, only change if you know what you are doing, ", + ) + + @pydantic.validator("sharded_table_pattern") + def sharded_table_pattern_is_a_valid_regexp(cls, v): + try: + re.compile(v) + except Exception as e: + raise ConfigurationError( + f"sharded_table_pattern configuration pattern is invalid. The exception was: {e}" + ) + return diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source_config/csv_enricher.py new file mode 100644 index 00000000000000..eb1ec323885695 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source_config/csv_enricher.py @@ -0,0 +1,38 @@ +from typing import Any, Dict + +import pydantic + +from datahub.configuration.common import ConfigModel, ConfigurationError + + +class CSVEnricherConfig(ConfigModel): + filename: str = pydantic.Field(description="Path to CSV file to ingest") + write_semantics: str = pydantic.Field( + default="PATCH", + description='Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be "PATCH" or "OVERRIDE"', + ) + delimiter: str = pydantic.Field( + default=",", description="Delimiter to use when parsing CSV" + ) + array_delimiter: str = pydantic.Field( + default="|", + description="Delimiter to use when parsing array fields (tags, terms and owners)", + ) + + @pydantic.validator("write_semantics") + def validate_write_semantics(cls, write_semantics: str) -> str: + if write_semantics.lower() not in {"patch", "override"}: + raise ConfigurationError( + "write_semantics cannot be any other value than PATCH or OVERRIDE. Default value is PATCH. " + "For PATCH semantics consider using the datahub-rest sink or " + "provide a datahub_api: configuration on your ingestion recipe" + ) + return write_semantics + + @pydantic.validator("array_delimiter") + def validator_diff(cls, array_delimiter: str, values: Dict[str, Any]) -> str: + if array_delimiter == values["delimiter"]: + raise ConfigurationError( + "array_delimiter and delimiter are the same. Please choose different delimiters." + ) + return array_delimiter diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source_config/pulsar.py index e21c6fc3ea42ba..e1bdf072787cd5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/pulsar.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/pulsar.py @@ -82,7 +82,7 @@ class PulsarSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigBase): ) exclude_individual_partitions: bool = Field( default=True, - description="Extract each individual partitioned topic. e.g. when turned off a topic with 100 partitions will result in 100 Datesets.", + description="Extract each individual partitioned topic. e.g. when turned off a topic with 100 partitions will result in 100 Datasets.", ) tenants: List[str] = Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/bigquery.py index 75789bb3e8cbca..989169844492b4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/bigquery.py @@ -18,11 +18,11 @@ class BigQueryConfig(BigQueryBaseConfig, BaseTimeWindowConfig, SQLAlchemyConfig) scheme: str = "bigquery" project_id: Optional[str] = pydantic.Field( default=None, - description="Project ID to ingest from. If not specified, will infer from environment.", + description="Project ID where you have rights to run queries and create tables. If `storage_project_id` is not specified then it is assumed this is the same project where data is stored. If not specified, will infer from environment.", ) - lineage_client_project_id: Optional[str] = pydantic.Field( + storage_project_id: Optional[str] = pydantic.Field( default=None, - description="If you want to use a different ProjectId for the lineage collection you can set it here.", + description="If your data is stored in a different project where you don't have rights to run jobs and create tables then specify this field. The same service account must have read rights in this GCP project and write rights in `project_id`.", ) log_page_size: pydantic.PositiveInt = pydantic.Field( default=1000, @@ -57,10 +57,6 @@ class BigQueryConfig(BigQueryBaseConfig, BaseTimeWindowConfig, SQLAlchemyConfig) description="Whether to read date sharded tables or time partitioned tables when extracting usage from exported audit logs.", ) _credentials_path: Optional[str] = pydantic.PrivateAttr(None) - temp_table_dataset_prefix: str = pydantic.Field( - default="_", - description="If you are creating temp tables in a dataset with a particular prefix you can use this config to set the prefix for the dataset. This is to support workflows from before bigquery's introduction of temp tables. By default we use `_` because of datasets that begin with an underscore are hidden by default https://cloud.google.com/bigquery/docs/datasets#dataset-naming.", - ) use_v2_audit_metadata: Optional[bool] = pydantic.Field( default=False, description="Whether to ingest logs using the v2 format." ) @@ -79,7 +75,9 @@ def __init__(self, **data: Any): ) os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self._credentials_path - def get_sql_alchemy_url(self): + def get_sql_alchemy_url(self, run_on_compute: bool = False) -> str: + if self.storage_project_id and not run_on_compute: + return f"{self.scheme}://{self.storage_project_id}" if self.project_id: return f"{self.scheme}://{self.project_id}" # When project_id is not set, we will attempt to detect the project ID @@ -98,6 +96,16 @@ def bigquery_doesnt_need_platform_instance(cls, v): def validate_that_bigquery_audit_metadata_datasets_is_correctly_configured( cls, values: Dict[str, Any] ) -> Dict[str, Any]: + profiling = values.get("profiling") + if ( + values.get("storage_project_id") + and profiling is not None + and profiling.enabled + and not profiling.bigquery_temp_table_schema + ): + raise ConfigurationError( + "If storage project is being used with profiling then bigquery_temp_table_schema needs to be set to a dataset in the compute project" + ) if ( values.get("use_exported_bigquery_audit_metadata") and not values.get("use_v2_audit_metadata") diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py index b00ac9cdfb41b8..713398d46aa96c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py @@ -90,7 +90,7 @@ class SnowflakeProvisionRoleConfig(ConfigModel): @pydantic.validator("admin_username", always=True) def username_not_empty(cls, v, values, **kwargs): v_str: str = str(v) - if v_str.strip() == "": + if not v_str.strip(): raise ValueError("username is empty") return v @@ -180,60 +180,55 @@ def authenticator_type_is_valid(cls, v, values, field): f"unsupported authenticator type '{v}' was provided," f" use one of {list(VALID_AUTH_TYPES.keys())}" ) - else: - if v == "KEY_PAIR_AUTHENTICATOR": - # If we are using key pair auth, we need the private key path and password to be set - if values.get("private_key_path") is None: - raise ValueError( - f"'private_key_path' was none " - f"but should be set when using {v} authentication" - ) - elif v == "OAUTH_AUTHENTICATOR": - if values.get("oauth_config") is None: - raise ValueError( - f"'oauth_config' is none but should be set when using {v} authentication" - ) - if values.get("oauth_config").provider is None: - raise ValueError( - f"'oauth_config.provider' is none " - f"but should be set when using {v} authentication" - ) - if values.get("oauth_config").client_id is None: - raise ValueError( - f"'oauth_config.client_id' is none " - f"but should be set when using {v} authentication" - ) - if values.get("oauth_config").scopes is None: + if v == "KEY_PAIR_AUTHENTICATOR": + # If we are using key pair auth, we need the private key path and password to be set + if values.get("private_key_path") is None: + raise ValueError( + f"'private_key_path' was none " + f"but should be set when using {v} authentication" + ) + elif v == "OAUTH_AUTHENTICATOR": + if values.get("oauth_config") is None: + raise ValueError( + f"'oauth_config' is none but should be set when using {v} authentication" + ) + if values.get("oauth_config").provider is None: + raise ValueError( + f"'oauth_config.provider' is none " + f"but should be set when using {v} authentication" + ) + if values.get("oauth_config").client_id is None: + raise ValueError( + f"'oauth_config.client_id' is none " + f"but should be set when using {v} authentication" + ) + if values.get("oauth_config").scopes is None: + raise ValueError( + f"'oauth_config.scopes' was none " + f"but should be set when using {v} authentication" + ) + if values.get("oauth_config").authority_url is None: + raise ValueError( + f"'oauth_config.authority_url' was none " + f"but should be set when using {v} authentication" + ) + if values.get("oauth_config").use_certificate is True: + if values.get("oauth_config").base64_encoded_oauth_private_key is None: raise ValueError( - f"'oauth_config.scopes' was none " - f"but should be set when using {v} authentication" + "'base64_encoded_oauth_private_key' was none " + "but should be set when using certificate for oauth_config" ) - if values.get("oauth_config").authority_url is None: + if values.get("oauth").base64_encoded_oauth_public_key is None: raise ValueError( - f"'oauth_config.authority_url' was none " - f"but should be set when using {v} authentication" + "'base64_encoded_oauth_public_key' was none" + "but should be set when using use_certificate true for oauth_config" ) - if values.get("oauth_config").use_certificate is True: - if ( - values.get("oauth_config").base64_encoded_oauth_private_key - is None - ): - raise ValueError( - "'base64_encoded_oauth_private_key' was none " - "but should be set when using certificate for oauth_config" - ) - if values.get("oauth").base64_encoded_oauth_public_key is None: - raise ValueError( - "'base64_encoded_oauth_public_key' was none" - "but should be set when using use_certificate true for oauth_config" - ) - else: - if values.get("oauth_config").client_secret is None: - raise ValueError( - "'oauth_config.client_secret' was none " - "but should be set when using use_certificate false for oauth_config" - ) - logger.info(f"using authenticator type '{v}'") + elif values.get("oauth_config").client_secret is None: + raise ValueError( + "'oauth_config.client_secret' was none " + "but should be set when using use_certificate false for oauth_config" + ) + logger.info(f"using authenticator type '{v}'") return v @pydantic.validator("include_view_lineage") @@ -244,35 +239,6 @@ def validate_include_view_lineage(cls, v, values): ) return v - def get_oauth_connection(self): - assert ( - self.oauth_config - ), "oauth_config should be provided if using oauth based authentication" - generator = OauthTokenGenerator( - self.oauth_config.client_id, - self.oauth_config.authority_url, - self.oauth_config.provider, - ) - if self.oauth_config.use_certificate is True: - response = generator.get_token_with_certificate( - private_key_content=str(self.oauth_config.encoded_oauth_public_key), - public_key_content=str(self.oauth_config.encoded_oauth_private_key), - scopes=self.oauth_config.scopes, - ) - else: - response = generator.get_token_with_secret( - secret=str(self.oauth_config.client_secret), - scopes=self.oauth_config.scopes, - ) - token = response["access_token"] - return snowflake.connector.connect( - user=self.username, - account=self.account_id, - authenticator="oauth", - token=token, - warehouse=self.warehouse, - ) - def get_sql_alchemy_url( self, database: Optional[str] = None, @@ -354,3 +320,79 @@ def get_options(self) -> dict: options_connect_args.update(self.options.get("connect_args", {})) self.options["connect_args"] = options_connect_args return self.options + + def get_oauth_connection(self): + assert ( + self.oauth_config + ), "oauth_config should be provided if using oauth based authentication" + generator = OauthTokenGenerator( + self.oauth_config.client_id, + self.oauth_config.authority_url, + self.oauth_config.provider, + ) + if self.oauth_config.use_certificate is True: + response = generator.get_token_with_certificate( + private_key_content=str(self.oauth_config.encoded_oauth_public_key), + public_key_content=str(self.oauth_config.encoded_oauth_private_key), + scopes=self.oauth_config.scopes, + ) + else: + response = generator.get_token_with_secret( + secret=str(self.oauth_config.client_secret), + scopes=self.oauth_config.scopes, + ) + token = response["access_token"] + connect_args = self.get_options()["connect_args"] + return snowflake.connector.connect( + user=self.username, + account=self.account_id, + token=token, + warehouse=self.warehouse, + authenticator=VALID_AUTH_TYPES.get(self.authentication_type), + application=APPLICATION_NAME, + **connect_args, + ) + + def get_key_pair_connection(self) -> snowflake.connector.SnowflakeConnection: + connect_args = self.get_options()["connect_args"] + + return snowflake.connector.connect( + user=self.username, + account=self.account_id, + warehouse=self.warehouse, + role=self.role, + authenticator=VALID_AUTH_TYPES.get(self.authentication_type), + application=APPLICATION_NAME, + **connect_args, + ) + + def get_connection(self) -> snowflake.connector.SnowflakeConnection: + connect_args = self.get_options()["connect_args"] + if self.authentication_type == "DEFAULT_AUTHENTICATOR": + return snowflake.connector.connect( + user=self.username, + password=self.password.get_secret_value() if self.password else None, + account=self.account_id, + warehouse=self.warehouse, + role=self.role, + application=APPLICATION_NAME, + **connect_args, + ) + elif self.authentication_type == "OAUTH_AUTHENTICATOR": + return self.get_oauth_connection() + elif self.authentication_type == "KEY_PAIR_AUTHENTICATOR": + return self.get_key_pair_connection() + elif self.authentication_type == "EXTERNAL_BROWSER_AUTHENTICATOR": + return snowflake.connector.connect( + user=self.username, + password=self.password.get_secret_value() if self.password else None, + account=self.account_id, + warehouse=self.warehouse, + role=self.role, + authenticator=VALID_AUTH_TYPES.get(self.authentication_type), + application=APPLICATION_NAME, + **connect_args, + ) + else: + # not expected to be here + raise Exception("Not expected to be here.") diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py index 05dc636d312c2e..9abee691ca9bf8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/usage/bigquery_usage.py @@ -114,7 +114,7 @@ class BigQueryUsageConfig(BigQueryBaseConfig, DatasetSourceConfigBase, BaseUsage credential: Optional[BigQueryCredential] = pydantic.Field( default=None, - description="Bigquery credential. Required if GOOGLE_APPLICATION_CREDENTIALS enviroment variable is not set. See this example recipe for details", + description="Bigquery credential. Required if GOOGLE_APPLICATION_CREDENTIALS environment variable is not set. See this example recipe for details", ) _credentials_path: Optional[str] = pydantic.PrivateAttr(None) temp_table_dataset_prefix: str = pydantic.Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py index 5083fa39d49a0f..066c0ee0c080d7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source_report/sql/bigquery.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Dict, Optional +from typing import Dict, List, Optional import pydantic @@ -12,6 +12,7 @@ class BigQueryReport(SQLSourceReport): num_total_lineage_entries: Optional[int] = None num_skipped_lineage_entries_missing_data: Optional[int] = None num_skipped_lineage_entries_not_allowed: Optional[int] = None + num_skipped_lineage_entries_sql_parser_failure: Optional[int] = None num_skipped_lineage_entries_other: Optional[int] = None num_total_log_entries: Optional[int] = None num_parsed_log_entires: Optional[int] = None @@ -32,3 +33,7 @@ class BigQueryReport(SQLSourceReport): audit_end_time: Optional[str] = None upstream_lineage: Dict = field(default_factory=dict) partition_info: Dict[str, str] = field(default_factory=dict) + table_metadata: Dict[str, List[str]] = field(default_factory=dict) + profile_table_selection_criteria: Dict[str, str] = field(default_factory=dict) + selected_profile_tables: Dict[str, List[str]] = field(default_factory=dict) + invalid_partition_ids: Dict[str, str] = field(default_factory=dict) diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py index 7235fea3b3b709..89c715efb1130f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py @@ -36,3 +36,6 @@ class SnowflakeReport(BaseSnowflakeReport, SQLSourceReport): role: str = "" check_role_grants: Optional[bool] = None role_grants: List[str] = field(default_factory=list) + + profile_if_updated_since: Optional[datetime] = None + profile_candidates: Dict[str, List[str]] = field(default_factory=dict) diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py index 89cb27fb93170e..506153087cdbd8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py @@ -25,6 +25,12 @@ class BigQueryUsageSourceReport(SourceReport): log_entry_end_time: Optional[str] = None num_usage_workunits_emitted: Optional[int] = None num_operational_stats_workunits_emitted: Optional[int] = None + read_reasons_stat: Counter[str] = dataclasses.field( + default_factory=collections.Counter + ) + operation_types_stat: Counter[str] = dataclasses.field( + default_factory=collections.Counter + ) def report_dropped(self, key: str) -> None: self.dropped_table[key] += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py index ecc1dcfc5fd31f..a2b1c135020658 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py @@ -1,6 +1,6 @@ import logging from abc import ABCMeta, abstractmethod -from typing import Any, Dict, Iterable, List, Optional, Type, Union +from typing import Any, Dict, Iterable, List, Optional, Type, Union, cast import datahub.emitter.mce_builder from datahub.emitter.mce_builder import Aspect @@ -17,6 +17,7 @@ DatasetPropertiesClass, DatasetSnapshotClass, DatasetUpstreamLineageClass, + DomainsClass, EditableDatasetPropertiesClass, EditableSchemaMetadataClass, GlobalTagsClass, @@ -29,6 +30,7 @@ StatusClass, UpstreamLineageClass, ViewPropertiesClass, + _Aspect, ) from datahub.utilities.urns.urn import Urn @@ -41,6 +43,7 @@ class SnapshotAspectRegistry: def __init__(self): self.aspect_name_type_mapping = { "ownership": OwnershipClass, + "domains": DomainsClass, "globalTags": GlobalTagsClass, "datasetProperties": DatasetPropertiesClass, "editableDatasetProperties": EditableDatasetPropertiesClass, @@ -132,8 +135,8 @@ def _should_process( return True # fall through, no entity type matched return False - elif isinstance(record, MetadataChangeProposalWrapper) or isinstance( - record, MetadataChangeProposalClass + elif isinstance( + record, (MetadataChangeProposalWrapper, MetadataChangeProposalClass) ): return record.entityType in entity_types @@ -215,7 +218,7 @@ def _transform_or_record_mcp( transformed_aspect = self.transform_aspect( entity_urn=envelope.record.entityUrn, aspect_name=envelope.record.aspectName, - aspect=envelope.record.aspect, + aspect=cast(_Aspect, envelope.record.aspect), ) self._mark_processed(envelope.record.entityUrn) if transformed_aspect is None: diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py new file mode 100644 index 00000000000000..5ed7d7d7616b3d --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_domain.py @@ -0,0 +1,169 @@ +from typing import Callable, List, Optional, Union, cast + +from datahub.configuration.common import ( + ConfigurationError, + KeyValuePattern, + TransformerSemantics, + TransformerSemanticsConfigModel, +) +from datahub.configuration.import_resolver import pydantic_resolve_key +from datahub.emitter.mce_builder import Aspect +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.transformer.dataset_transformer import DatasetDomainTransformer +from datahub.metadata.schema_classes import DomainsClass +from datahub.utilities.registries.domain_registry import DomainRegistry + + +class AddDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel): + get_domains_to_add: Union[ + Callable[[str], DomainsClass], + Callable[[str], DomainsClass], + ] + + _resolve_domain_fn = pydantic_resolve_key("get_domains_to_add") + + +class SimpleDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel): + domain_urns: List[str] + + +class PatternDatasetDomainSemanticsConfig(TransformerSemanticsConfigModel): + domain_pattern: KeyValuePattern = KeyValuePattern.all() + + +class AddDatasetDomain(DatasetDomainTransformer): + """Transformer that adds domains to datasets according to a callback function.""" + + ctx: PipelineContext + config: AddDatasetDomainSemanticsConfig + + def __init__(self, config: AddDatasetDomainSemanticsConfig, ctx: PipelineContext): + super().__init__() + self.ctx = ctx + self.config = config + + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> "AddDatasetDomain": + config = AddDatasetDomainSemanticsConfig.parse_obj(config_dict) + return cls(config, ctx) + + @staticmethod + def get_domain_class( + graph: Optional[DataHubGraph], domains: List[str] + ) -> DomainsClass: + domain_registry: DomainRegistry = DomainRegistry( + cached_domains=[k for k in domains], graph=graph + ) + domain_class = DomainsClass( + domains=[domain_registry.get_domain_urn(domain) for domain in domains] + ) + return domain_class + + @staticmethod + def get_domains_to_set( + graph: DataHubGraph, urn: str, mce_domain: Optional[DomainsClass] + ) -> Optional[DomainsClass]: + if not mce_domain or not mce_domain.domains: + # nothing to add, no need to consult server + return None + + server_domain = graph.get_domain(entity_urn=urn) + if server_domain: + # compute patch + # we only include domain who are not present in the server domain list + domains_to_add: List[str] = [] + for domain in mce_domain.domains: + if domain not in server_domain.domains: + domains_to_add.append(domain) + + mce_domain.domains.extend(server_domain.domains) + mce_domain.domains.extend(domains_to_add) + + return mce_domain + + def transform_aspect( + self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect] + ) -> Optional[Aspect]: + + domain_aspect: DomainsClass = DomainsClass(domains=[]) + # Check if we have received existing aspect + if aspect is not None: + domain_aspect.domains.extend(cast(DomainsClass, aspect).domains) + + domain_to_add = self.config.get_domains_to_add(entity_urn) + + domain_aspect.domains.extend(domain_to_add.domains) + + if self.config.semantics == TransformerSemantics.PATCH: + assert self.ctx.graph + patch_domain_aspect: Optional[ + DomainsClass + ] = AddDatasetDomain.get_domains_to_set( + self.ctx.graph, entity_urn, domain_aspect + ) + # This will pass the mypy lint + domain_aspect = ( + patch_domain_aspect + if patch_domain_aspect is not None + else domain_aspect + ) + + return cast(Optional[Aspect], domain_aspect) + + +class SimpleAddDatasetDomain(AddDatasetDomain): + """Transformer that adds a specified set of domains to each dataset.""" + + def __init__( + self, config: SimpleDatasetDomainSemanticsConfig, ctx: PipelineContext + ): + if ctx.graph is None: + raise ConfigurationError( + "AddDatasetDomain requires a datahub_api to connect to. Consider using the datahub-rest sink or provide a datahub_api: configuration on your ingestion recipe" + ) + + domains = AddDatasetDomain.get_domain_class(ctx.graph, config.domain_urns) + generic_config = AddDatasetDomainSemanticsConfig( + get_domains_to_add=lambda _: domains, + semantics=config.semantics, + ) + super().__init__(generic_config, ctx) + + @classmethod + def create( + cls, config_dict: dict, ctx: PipelineContext + ) -> "SimpleAddDatasetDomain": + config = SimpleDatasetDomainSemanticsConfig.parse_obj(config_dict) + return cls(config, ctx) + + +class PatternAddDatasetDomain(AddDatasetDomain): + """Transformer that adds a specified set of domains to each dataset.""" + + def __init__( + self, config: PatternDatasetDomainSemanticsConfig, ctx: PipelineContext + ): + if ctx.graph is None: + raise ConfigurationError( + "AddDatasetDomain requires a datahub_api to connect to. Consider using the datahub-rest sink or provide a datahub_api: configuration on your ingestion recipe" + ) + + domain_pattern = config.domain_pattern + + def resolve_domain(domain_urn: str) -> DomainsClass: + domains = domain_pattern.value(domain_urn) + return self.get_domain_class(ctx.graph, domains) + + generic_config = AddDatasetDomainSemanticsConfig( + get_domains_to_add=resolve_domain, + semantics=config.semantics, + ) + super().__init__(generic_config, ctx) + + @classmethod + def create( + cls, config_dict: dict, ctx: PipelineContext + ) -> "PatternAddDatasetDomain": + config = PatternDatasetDomainSemanticsConfig.parse_obj(config_dict) + return cls(config, ctx) diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py index e7c2806b8cee0a..c2e6ddf141c5bf 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/dataset_transformer.py @@ -1,5 +1,5 @@ import logging -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from typing import List, Optional from deprecated import deprecated @@ -59,11 +59,27 @@ def transform_aspect( # not marked as @abstractmethod to avoid impacting transf ) +# TODO: rename DatasetTransformerV2 to DatasetTransformer after upgrading all existing dataset transformer +class DatasetTransformerV2(BaseTransformer, SingleAspectTransformer, metaclass=ABCMeta): + """Transformer that does transforms sequentially on each dataset.""" + + def __init__(self): + super().__init__() + + def entity_types(self) -> List[str]: + return ["dataset"] + + class DatasetOwnershipTransformer(DatasetTransformer, SingleAspectTransformer): def aspect_name(self) -> str: return "ownership" +class DatasetDomainTransformer(DatasetTransformerV2, SingleAspectTransformer): + def aspect_name(self) -> str: + return "domains" + + class DatasetStatusTransformer(DatasetTransformer, SingleAspectTransformer): def aspect_name(self) -> str: return "status" diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/mark_dataset_status.py b/metadata-ingestion/src/datahub/ingestion/transformer/mark_dataset_status.py index bae8d0e07a80ab..d833e9bcc75a64 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/mark_dataset_status.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/mark_dataset_status.py @@ -40,6 +40,6 @@ def transform_aspect( self, entity_urn: str, aspect_name: str, aspect: Optional[builder.Aspect] ) -> Optional[builder.Aspect]: assert aspect is None or isinstance(aspect, StatusClass) - status_aspect: StatusClass = aspect if aspect else StatusClass(removed=None) + status_aspect: StatusClass = aspect or StatusClass(removed=None) status_aspect.removed = self.config.removed return status_aspect # type: ignore diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/transform_registry.py b/metadata-ingestion/src/datahub/ingestion/transformer/transform_registry.py index a9083f54d70f29..972d17f03a235c 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/transform_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/transform_registry.py @@ -1,5 +1,6 @@ from datahub.ingestion.api.registry import PluginRegistry from datahub.ingestion.api.transform import Transformer +from datahub.ingestion.transformer import dataset_domain from datahub.ingestion.transformer.add_dataset_browse_path import ( AddDatasetBrowsePathTransformer, ) @@ -47,6 +48,15 @@ transform_registry.register("simple_add_dataset_ownership", SimpleAddDatasetOwnership) transform_registry.register("pattern_add_dataset_ownership", PatternAddDatasetOwnership) +transform_registry.register("add_dataset_domain", dataset_domain.AddDatasetDomain) +transform_registry.register( + "simple_add_dataset_domain", dataset_domain.SimpleAddDatasetDomain +) +transform_registry.register( + "pattern_add_dataset_domain", dataset_domain.PatternAddDatasetDomain +) + + transform_registry.register("add_dataset_tags", AddDatasetTags) transform_registry.register("simple_add_dataset_tags", SimpleAddDatasetTags) diff --git a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py index 98b344c0a06cc7..5dca3541493156 100644 --- a/metadata-ingestion/src/datahub/integrations/great_expectations/action.py +++ b/metadata-ingestion/src/datahub/integrations/great_expectations/action.py @@ -190,12 +190,11 @@ def _run( result = "DataHub notification succeeded" except Exception as e: result = "DataHub notification failed" - if self.graceful_exceptions: - logger.error(e) - logger.info("Supressing error because graceful_exceptions is set") - else: + if not self.graceful_exceptions: raise + logger.error(e) + logger.info("Suppressing error because graceful_exceptions is set") return {"datahub_notification_result": result} def get_assertions_with_results( @@ -224,7 +223,7 @@ def get_assertions_with_results( for result in validation_result_suite.results: expectation_config = result["expectation_config"] expectation_type = expectation_config["expectation_type"] - success = True if result["success"] else False + success = bool(result["success"]) kwargs = { k: v for k, v in expectation_config["kwargs"].items() if k != "batch_id" } @@ -271,8 +270,6 @@ def get_assertions_with_results( # TODO: Understand why their run time is incorrect. run_time = run_id.run_time.astimezone(timezone.utc) - assertionResults = [] - evaluation_parameters = ( { k: convert_to_string(v) @@ -328,8 +325,7 @@ def get_assertions_with_results( ) if ds.get("partitionSpec") is not None: assertionResult.partitionSpec = ds.get("partitionSpec") - assertionResults.append(assertionResult) - + assertionResults = [assertionResult] assertions_with_results.append( { "assertionUrn": assertionUrn, @@ -631,8 +627,9 @@ def get_dataset_partitions(self, batch_identifier, data_asset): ].batch_request.runtime_parameters["query"] partitionSpec = PartitionSpecClass( type=PartitionTypeClass.QUERY, - partition="Query_" + builder.datahub_guid(query), + partition=f"Query_{builder.datahub_guid(query)}", ) + batchSpec = BatchSpec( nativeBatchId=batch_identifier, query=query, @@ -678,9 +675,9 @@ def get_dataset_partitions(self, batch_identifier, data_asset): return dataset_partitions def get_platform_instance(self, datasource_name): - if self.platform_instance_map and datasource_name in self.platform_instance_map: - return self.platform_instance_map[datasource_name] if self.platform_instance_map: + if datasource_name in self.platform_instance_map: + return self.platform_instance_map[datasource_name] warn( f"Datasource {datasource_name} is not present in platform_instance_map" ) @@ -698,21 +695,21 @@ def make_dataset_urn_from_sqlalchemy_uri( schema_name, table_name = table_name.split(".")[-2:] if data_platform in ["redshift", "postgres"]: - schema_name = schema_name if schema_name else "public" + schema_name = schema_name or "public" if url_instance.database is None: warn( f"DataHubValidationAction failed to locate database name for {data_platform}." ) return None - schema_name = "{}.{}".format(url_instance.database, schema_name) + schema_name = f"{url_instance.database}.{schema_name}" elif data_platform == "mssql": - schema_name = schema_name if schema_name else "dbo" + schema_name = schema_name or "dbo" if url_instance.database is None: warn( f"DataHubValidationAction failed to locate database name for {data_platform}." ) return None - schema_name = "{}.{}".format(url_instance.database, schema_name) + schema_name = f"{url_instance.database}.{schema_name}" elif data_platform in ["trino", "snowflake"]: if schema_name is None or url_instance.database is None: warn( @@ -738,16 +735,16 @@ def make_dataset_urn_from_sqlalchemy_uri( ) ) return None - schema_name = "{}.{}".format(url_instance.host, url_instance.database) + schema_name = f"{url_instance.host}.{url_instance.database}" - schema_name = schema_name if schema_name else url_instance.database + schema_name = schema_name or url_instance.database if schema_name is None: warn( f"DataHubValidationAction failed to locate schema name for {data_platform}." ) return None - dataset_name = "{}.{}".format(schema_name, table_name) + dataset_name = f"{schema_name}.{table_name}" dataset_urn = builder.make_dataset_urn_with_platform_instance( platform=data_platform, diff --git a/metadata-ingestion/src/datahub/telemetry/stats.py b/metadata-ingestion/src/datahub/telemetry/stats.py index ea48aab14c77db..bf98bd72b574ce 100644 --- a/metadata-ingestion/src/datahub/telemetry/stats.py +++ b/metadata-ingestion/src/datahub/telemetry/stats.py @@ -9,12 +9,12 @@ def __lt__(self, __other: Any) -> Any: ... -SupportsComparisonT = TypeVar("SupportsComparisonT", bound=SupportsLT) # noqa: Y001 +_SupportsComparisonT = TypeVar("_SupportsComparisonT", bound=SupportsLT) def calculate_percentiles( - data: List[SupportsComparisonT], percentiles: List[int] -) -> Dict[int, SupportsComparisonT]: + data: List[_SupportsComparisonT], percentiles: List[int] +) -> Dict[int, _SupportsComparisonT]: size = len(data) if size == 0: @@ -27,9 +27,7 @@ def calculate_percentiles( min(i, size - 1) for i in percentile_indices ] # in case of rounding errors - values = {p: data_sorted[i] for p, i in zip(percentiles, percentile_indices)} - - return values + return {p: data_sorted[i] for p, i in zip(percentiles, percentile_indices)} def discretize(statistic: Union[float, int]) -> int: diff --git a/metadata-ingestion/src/datahub/telemetry/telemetry.py b/metadata-ingestion/src/datahub/telemetry/telemetry.py index b95df169414320..470e3ffeb89553 100644 --- a/metadata-ingestion/src/datahub/telemetry/telemetry.py +++ b/metadata-ingestion/src/datahub/telemetry/telemetry.py @@ -11,11 +11,12 @@ from mixpanel import Consumer, Mixpanel import datahub as datahub_package +from datahub.cli.cli_utils import DATAHUB_ROOT_FOLDER from datahub.ingestion.graph.client import DataHubGraph logger = logging.getLogger(__name__) -DATAHUB_FOLDER = Path(os.path.expanduser("~/.datahub")) +DATAHUB_FOLDER = Path(DATAHUB_ROOT_FOLDER) CONFIG_FILE = DATAHUB_FOLDER / "telemetry-config.json" @@ -262,18 +263,18 @@ def _server_props(self, server: Optional[DataHubGraph]) -> Dict[str, str]: T = TypeVar("T") -def set_telemetry_enable(enable: bool) -> Any: - telemetry_instance.enabled = enable - if not enable: - logger.info("Disabling Telemetry locally due to server config") - telemetry_instance.update_config() +def suppress_telemetry() -> Any: + """disables telemetry for this invocation, doesn't affect persistent client settings""" + if telemetry_instance.enabled: + logger.debug("Disabling telemetry locally due to server config") + telemetry_instance.enabled = False def get_full_class_name(obj): module = obj.__class__.__module__ if module is None or module == str.__class__.__module__: return obj.__class__.__name__ - return module + "." + obj.__class__.__name__ + return f"{module}.{obj.__class__.__name__}" def with_telemetry(func: Callable[..., T]) -> Callable[..., T]: diff --git a/metadata-ingestion/src/datahub/upgrade/upgrade.py b/metadata-ingestion/src/datahub/upgrade/upgrade.py index 046f32202d83bd..32c55284870287 100644 --- a/metadata-ingestion/src/datahub/upgrade/upgrade.py +++ b/metadata-ingestion/src/datahub/upgrade/upgrade.py @@ -1,17 +1,20 @@ +import asyncio +import contextlib +import functools import logging from datetime import datetime, timedelta, timezone from functools import wraps -from typing import Any, Callable, Optional, TypeVar +from typing import Any, Callable, Optional, Tuple, TypeVar +import aiohttp import humanfriendly -import requests from packaging.version import Version from pydantic import BaseModel from termcolor import colored from datahub import __version__ from datahub.cli import cli_utils -from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.ingestion.graph.client import DataHubGraph log = logging.getLogger(__name__) @@ -40,121 +43,155 @@ class DataHubVersionStats(BaseModel): client: ClientVersionStats -def retrieve_versions( # noqa: C901 - server: Optional[DataHubGraph] = None, -) -> Optional[DataHubVersionStats]: - +async def get_client_version_stats(): current_version_string = __version__ current_version = Version(current_version_string) client_version_stats: ClientVersionStats = ClientVersionStats( current=VersionStats(version=current_version, release_date=None), latest=None ) - server_version_stats: Optional[ServerVersionStats] = None - - try: - response = requests.get("https://pypi.org/pypi/acryl_datahub/json") - if response.ok: - response_json = response.json() - releases = response_json.get("releases", []) - sorted_releases = sorted(releases.keys(), key=lambda x: Version(x)) - latest_cli_release_string = [x for x in sorted_releases if "rc" not in x][ - -1 - ] - latest_cli_release = Version(latest_cli_release_string) - current_version_info = releases.get(current_version_string) - current_version_date = None - if current_version_info: - current_version_date = datetime.strptime( - current_version_info[0].get("upload_time"), "%Y-%m-%dT%H:%M:%S" - ) - latest_release_info = releases.get(latest_cli_release_string) - latest_version_date = None - if latest_release_info: - latest_version_date = datetime.strptime( - latest_release_info[0].get("upload_time"), "%Y-%m-%dT%H:%M:%S" + async with aiohttp.ClientSession() as session: + pypi_url = "https://pypi.org/pypi/acryl_datahub/json" + async with session.get(pypi_url) as resp: + response_json = await resp.json() + try: + releases = response_json.get("releases", []) + sorted_releases = sorted(releases.keys(), key=lambda x: Version(x)) + latest_cli_release_string = [ + x for x in sorted_releases if "rc" not in x + ][-1] + latest_cli_release = Version(latest_cli_release_string) + current_version_info = releases.get(current_version_string) + current_version_date = None + if current_version_info: + current_version_date = datetime.strptime( + current_version_info[0].get("upload_time"), "%Y-%m-%dT%H:%M:%S" + ) + latest_release_info = releases.get(latest_cli_release_string) + latest_version_date = None + if latest_release_info: + latest_version_date = datetime.strptime( + latest_release_info[0].get("upload_time"), "%Y-%m-%dT%H:%M:%S" + ) + client_version_stats = ClientVersionStats( + current=VersionStats( + version=current_version, release_date=current_version_date + ), + latest=VersionStats( + version=latest_cli_release, release_date=latest_version_date + ), ) - client_version_stats = ClientVersionStats( - current=VersionStats( - version=current_version, release_date=current_version_date - ), - latest=VersionStats( - version=latest_cli_release, release_date=latest_version_date - ), - ) - except Exception as e: - log.debug(f"Failed to determine cli releases from pypi due to {e}") - pass + except Exception as e: + log.debug(f"Failed to determine cli releases from pypi due to {e}") + pass + return client_version_stats - latest_server_version: Optional[Version] = None - latest_server_date: Optional[datetime] = None - server_version: Optional[Version] = None - try: - gh_response = requests.get( - "https://api.github.com/repos/datahub-project/datahub/releases", - headers={"Accept": "application/vnd.github.v3+json"}, - ) - if gh_response.ok: - gh_response_json = gh_response.json() +async def get_github_stats(): + async with aiohttp.ClientSession( + headers={"Accept": "application/vnd.github.v3+json"} + ) as session: + gh_url = "https://api.github.com/repos/datahub-project/datahub/releases" + async with session.get(gh_url) as gh_response: + gh_response_json = await gh_response.json() latest_server_version = Version(gh_response_json[0].get("tag_name")) latest_server_date = gh_response_json[0].get("published_at") - except Exception as e: - log.debug("Failed to get release versions from github", e) + return (latest_server_version, latest_server_date) + + +async def get_server_config(gms_url: str, token: str) -> dict: + async with aiohttp.ClientSession( + headers={ + "X-RestLi-Protocol-Version": "2.0.0", + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + } + ) as session: + config_endpoint = f"{gms_url}/config" + async with session.get(config_endpoint) as dh_response: + dh_response_json = await dh_response.json() + return dh_response_json + +async def get_server_version_stats( + server: Optional[DataHubGraph] = None, +) -> Tuple[Optional[str], Optional[Version], Optional[datetime]]: + server_config = None if not server: try: # let's get the server from the cli config - host, token = cli_utils.get_host_and_token() - server = DataHubGraph(DatahubClientConfig(server=host, token=token)) + host, token = cli_utils.get_url_and_token() + server_config = await get_server_config(host, token) + log.debug(f"server_config:{server_config}") except Exception as e: log.debug("Failed to get a valid server", e) pass + else: + server_config = server.server_config server_type = None - if server: + server_version: Optional[Version] = None + current_server_release_date = None + if server_config: server_version_string = ( - server.server_config.get("versions", {}) - .get("linkedin/datahub", {}) - .get("version") + server_config.get("versions", {}).get("linkedin/datahub", {}).get("version") ) commit_hash = ( - server.server_config.get("versions", {}) - .get("linkedin/datahub", {}) - .get("commit") - ) - server_type = server.server_config.get("datahub", {}).get( - "serverType", "unknown" + server_config.get("versions", {}).get("linkedin/datahub", {}).get("commit") ) - current_server_release_date = None + server_type = server_config.get("datahub", {}).get("serverType", "unknown") if server_type == "quickstart" and commit_hash: - try: - # get the age of the commit from github - gh_commit_response = requests.get( - f"https://api.github.com/repos/datahub-project/datahub/commits/{commit_hash}", - headers={"Accept": "application/vnd.github.v3+json"}, - ) - if gh_commit_response.ok: + async with aiohttp.ClientSession( + headers={"Accept": "application/vnd.github.v3+json"} + ) as session: + gh_url = f"https://api.github.com/repos/datahub-project/datahub/commits/{commit_hash}" + async with session.get(gh_url) as gh_response: + gh_commit_response = await gh_response.json() current_server_release_date = datetime.strptime( - gh_commit_response.json()["commit"]["author"]["date"], + gh_commit_response["commit"]["author"]["date"], "%Y-%m-%dT%H:%M:%S%z", ) - except Exception as e: - log.debug(f"Failed to retrieve commit date due to {e}") - pass - if server_version_string and server_version_string.startswith("v"): server_version = Version(server_version_string[1:]) + return (server_type, server_version, current_server_release_date) + + +async def retrieve_version_stats( + server: Optional[DataHubGraph] = None, +) -> Optional[DataHubVersionStats]: + + client_version_stats_future = asyncio.ensure_future(get_client_version_stats()) + github_stats_future = asyncio.ensure_future(get_github_stats()) + server_config_future = asyncio.ensure_future(get_server_version_stats(server)) + tasks = [client_version_stats_future, github_stats_future, server_config_future] + + while len(tasks): + done, pending = await asyncio.wait(tasks, timeout=0.1) + for t in done: + if t == client_version_stats_future: + client_version_stats = t.result() + elif t == github_stats_future: + (last_server_version, last_server_date) = t.result() + elif t == server_config_future: + ( + current_server_type, + current_server_version, + current_server_release_date, + ) = server_config_future.result() + tasks.remove(t) + + server_version_stats = None + if current_server_version: server_version_stats = ServerVersionStats( current=VersionStats( - version=server_version, release_date=current_server_release_date + version=current_server_version, release_date=current_server_release_date ), latest=VersionStats( - version=latest_server_version, release_date=latest_server_date + version=last_server_version, release_date=last_server_date ) - if latest_server_version + if last_server_version else None, - current_server_type=server_type, + current_server_type=current_server_type, ) if client_version_stats and server_version_stats: @@ -213,7 +250,7 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i def maybe_print_upgrade_message( # noqa: C901 - server: Optional[DataHubGraph] = None, + version_stats: Optional[DataHubVersionStats], ) -> None: # noqa: C901 days_before_cli_stale = 7 days_before_quickstart_stale = 7 @@ -221,10 +258,12 @@ def maybe_print_upgrade_message( # noqa: C901 encourage_cli_upgrade = False client_server_compat = 0 encourage_quickstart_upgrade = False - try: - version_stats = retrieve_versions(server) + with contextlib.suppress(Exception): if not version_stats: + log.debug("No version stats found") return + else: + log.debug(f"Version stats found: {version_stats}") current_release_date = version_stats.client.current.release_date latest_release_date = ( version_stats.client.latest.release_date @@ -254,19 +293,22 @@ def maybe_print_upgrade_message( # noqa: C901 - version_stats.server.current.release_date ) if time_delta > timedelta(days=days_before_quickstart_stale): + log.debug( + f"will encourage upgrade due to server being old {version_stats.server.current.release_date},{time_delta}" + ) encourage_quickstart_upgrade = True if version_stats.server.latest and ( version_stats.server.latest.version > version_stats.server.current.version ): + log.debug( + f"Will encourage upgrade due to newer version of server {version_stats.server.latest.version} being available compared to {version_stats.server.current.version}" + ) encourage_quickstart_upgrade = True - except Exception: - pass - # Compute recommendations and print one if client_server_compat < 0: - try: + with contextlib.suppress(Exception): assert version_stats print( colored("❗Client-Server Incompatible❗", "yellow"), @@ -279,10 +321,8 @@ def maybe_print_upgrade_message( # noqa: C901 "cyan", ), ) - except Exception: - pass elif client_server_compat > 0: - try: + with contextlib.suppress(Exception): assert version_stats print( colored("❗Client-Server Incompatible❗", "red"), @@ -295,12 +335,8 @@ def maybe_print_upgrade_message( # noqa: C901 "cyan", ), ) - except Exception: - pass - - # we only encourage upgrades if we think client_server is currently compatible elif client_server_compat == 0 and encourage_cli_upgrade: - try: + with contextlib.suppress(Exception): print( colored("💡 Upgrade cli!", "yellow"), colored( @@ -308,9 +344,6 @@ def maybe_print_upgrade_message( # noqa: C901 "cyan", ), ) - except Exception: - pass - elif encourage_quickstart_upgrade: try: assert version_stats @@ -328,16 +361,29 @@ def maybe_print_upgrade_message( # noqa: C901 def check_upgrade(func: Callable[..., T]) -> Callable[..., T]: @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: + def async_wrapper(*args: Any, **kwargs: Any) -> Any: + async def run_inner_func(): + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, functools.partial(func, *args, **kwargs) + ) - res = func(*args, **kwargs) - try: - # ensure this cannot fail - maybe_print_upgrade_message() - except Exception as e: - log.debug(f"Failed to check for upgrade due to {e}") - pass + async def run_func_check_upgrade(): + version_stats_future = asyncio.ensure_future(retrieve_version_stats()) + the_one_future = asyncio.ensure_future(run_inner_func()) + ret = await the_one_future + + # the one future has returned + # we check the other futures quickly + try: + version_stats = await asyncio.wait_for(version_stats_future, 0.5) + maybe_print_upgrade_message(version_stats=version_stats) + except Exception: + log.debug("timed out waiting for version stats to be computed") + + return ret - return res + loop = asyncio.get_event_loop() + loop.run_until_complete(run_func_check_upgrade()) - return wrapper + return async_wrapper diff --git a/metadata-ingestion/src/datahub/utilities/checkpoint_state_util.py b/metadata-ingestion/src/datahub/utilities/checkpoint_state_util.py new file mode 100644 index 00000000000000..d069dedceb9ae9 --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/checkpoint_state_util.py @@ -0,0 +1,38 @@ +from typing import Iterable, List, Set + +from datahub.emitter.mce_builder import dataset_urn_to_key, make_dataset_urn + + +class CheckpointStateUtil: + """ + A utility class to provide common functionalities around Urn and CheckPointState of different DataHub entities + """ + + @staticmethod + def get_separator() -> str: + # Unique small string not allowed in URNs. + return "||" + + @staticmethod + def get_encoded_urns_not_in( + encoded_urns_1: List[str], encoded_urns_2: List[str] + ) -> Set[str]: + return set(encoded_urns_1) - set(encoded_urns_2) + + @staticmethod + def get_dataset_lightweight_repr(dataset_urn: str) -> str: + SEP = CheckpointStateUtil.get_separator() + key = dataset_urn_to_key(dataset_urn) + assert key is not None + return f"{key.platform}{SEP}{key.name}{SEP}{key.origin}" + + @staticmethod + def get_dataset_urns_not_in( + encoded_urns_1: List[str], encoded_urns_2: List[str] + ) -> Iterable[str]: + difference = CheckpointStateUtil.get_encoded_urns_not_in( + encoded_urns_1, encoded_urns_2 + ) + for encoded_urn in difference: + platform, name, env = encoded_urn.split(CheckpointStateUtil.get_separator()) + yield make_dataset_urn(platform, name, env) diff --git a/metadata-ingestion/src/datahub/utilities/hive_schema_to_avro.py b/metadata-ingestion/src/datahub/utilities/hive_schema_to_avro.py index fc9680ba642d42..6e8d8da5f3fb82 100644 --- a/metadata-ingestion/src/datahub/utilities/hive_schema_to_avro.py +++ b/metadata-ingestion/src/datahub/utilities/hive_schema_to_avro.py @@ -53,9 +53,12 @@ def _parse_datatype_string( parts = HiveColumnToAvroConverter._ignore_brackets_split(s[4:-1], ",") if len(parts) != 2: raise ValueError( - "The map type string format is: 'map', " - + "but got: %s" % s + ( + "The map type string format is: 'map', " + + f"but got: {s}" + ) ) + kt = HiveColumnToAvroConverter._parse_datatype_string(parts[0]) vt = HiveColumnToAvroConverter._parse_datatype_string(parts[1]) # keys are assumed to be strings in avro map @@ -103,9 +106,12 @@ def _parse_struct_fields_string(s: str, **kwargs: Any) -> Dict[str, object]: name_and_type = HiveColumnToAvroConverter._ignore_brackets_split(part, ":") if len(name_and_type) != 2: raise ValueError( - "The struct field string format is: 'field_name:field_type', " - + "but got: %s" % part + ( + "The struct field string format is: 'field_name:field_type', " + + f"but got: {part}" + ) ) + field_name = name_and_type[0].strip() if field_name.startswith("`"): if field_name[-1] != "`": @@ -117,16 +123,15 @@ def _parse_struct_fields_string(s: str, **kwargs: Any) -> Dict[str, object]: fields.append({"name": field_name, "type": field_type}) if kwargs.get("ustruct_seqn") is not None: - struct_name = "__structn_{}_{}".format( - kwargs["ustruct_seqn"], str(uuid.uuid4()).replace("-", "") - ) + struct_name = f'__structn_{kwargs["ustruct_seqn"]}_{str(uuid.uuid4()).replace("-", "")}' + else: - struct_name = "__struct_{}".format(str(uuid.uuid4()).replace("-", "")) + struct_name = f'__struct_{str(uuid.uuid4()).replace("-", "")}' return { "type": "record", "name": struct_name, "fields": fields, - "native_data_type": "struct<{}>".format(s), + "native_data_type": f"struct<{s}>", } @staticmethod @@ -193,7 +198,7 @@ def _ignore_brackets_split(s: str, separator: str) -> List[str]: buf += c elif c in HiveColumnToAvroConverter._BRACKETS.values(): if level == 0: - raise ValueError("Brackets are not correctly paired: %s" % s) + raise ValueError(f"Brackets are not correctly paired: {s}") level -= 1 buf += c elif c == separator and level > 0: @@ -205,7 +210,7 @@ def _ignore_brackets_split(s: str, separator: str) -> List[str]: buf += c if len(buf) == 0: - raise ValueError("The %s cannot be the last char: %s" % (separator, s)) + raise ValueError(f"The {separator} cannot be the last char: {s}") parts.append(buf) return parts diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py index 2b6c458db8d1ce..6212f1b001e002 100644 --- a/metadata-ingestion/src/datahub/utilities/mapping.py +++ b/metadata-ingestion/src/datahub/utilities/mapping.py @@ -1,9 +1,15 @@ import logging import re -from typing import Any, Dict, Optional +from typing import Any, Dict, Match, Optional, Union from datahub.emitter import mce_builder from datahub.emitter.mce_builder import OwnerType +from datahub.metadata.schema_classes import ( + OwnerClass, + OwnershipClass, + OwnershipSourceClass, + OwnershipTypeClass, +) class Constants: @@ -15,6 +21,7 @@ class Constants: TAG = "tag" TERM = "term" OWNER_TYPE = "owner_type" + OWNER_CATEGORY = "owner_category" MATCH = "match" USER_OWNER = "user" GROUP_OWNER = "group" @@ -77,7 +84,7 @@ def process(self, raw_props: Dict[str, Any]) -> Dict[str, Any]: # operation_type: the type of operation (add_tag, add_term, etc.) aspect_map: Dict[str, Any] = {} # map of aspect name to aspect object try: - operations_map: Dict[str, set] = {} + operations_map: Dict[str, Union[set, list]] = {} for operation_key in self.operation_defs: operation_type = self.operation_defs.get(operation_key, {}).get( Constants.OPERATION @@ -87,25 +94,36 @@ def process(self, raw_props: Dict[str, Any]) -> Dict[str, Any]: ) if not operation_type or not operation_config: continue - if self.is_match( + maybe_match = self.get_match( self.operation_defs[operation_key][Constants.MATCH], raw_props.get(operation_key), - ): + ) + if maybe_match is not None: operation = self.get_operation_value( - operation_key, operation_type, operation_config, raw_props + operation_key, operation_type, operation_config, maybe_match ) if operation: - operations_value_set: set = operations_map.get( - operation_type, set() - ) - operations_value_set.add(operation) - operations_map[operation_type] = operations_value_set + if isinstance(operation, str): + operations_value_set = operations_map.get( + operation_type, set() + ) + operations_value_set.add(operation) # type: ignore + operations_map[operation_type] = operations_value_set + else: + operations_value_list = operations_map.get( + operation_type, list() + ) + operations_value_list.append(operation) # type: ignore + operations_map[operation_type] = operations_value_list + aspect_map = self.convert_to_aspects(operations_map) except Exception as e: self.logger.error("Error while processing operation defs over raw_props", e) return aspect_map - def convert_to_aspects(self, operation_map: Dict[str, set]) -> Dict[str, Any]: + def convert_to_aspects( + self, operation_map: Dict[str, Union[set, list]] + ) -> Dict[str, Any]: aspect_map: Dict[str, Any] = {} if Constants.ADD_TAG_OPERATION in operation_map: tag_aspect = mce_builder.make_global_tag_aspect_with_tag_list( @@ -113,9 +131,20 @@ def convert_to_aspects(self, operation_map: Dict[str, set]) -> Dict[str, Any]: ) aspect_map[Constants.ADD_TAG_OPERATION] = tag_aspect if Constants.ADD_OWNER_OPERATION in operation_map: - owner_aspect = mce_builder.make_ownership_aspect_from_urn_list( - sorted(operation_map[Constants.ADD_OWNER_OPERATION]), - self.owner_source_type, + owner_aspect = OwnershipClass( + owners=[ + OwnerClass( + owner=x.get("urn"), + type=x.get("category"), + source=OwnershipSourceClass(type=self.owner_source_type) + if self.owner_source_type + else None, + ) + for x in sorted( + operation_map[Constants.ADD_OWNER_OPERATION], + key=lambda x: x["urn"], + ) + ] ) aspect_map[Constants.ADD_OWNER_OPERATION] = owner_aspect if Constants.ADD_TERM_OPERATION in operation_map: @@ -130,8 +159,22 @@ def get_operation_value( operation_key: str, operation_type: str, operation_config: Dict, - raw_props: Dict, - ) -> Optional[str]: + match: Match, + ) -> Optional[Union[str, Dict]]: + def _get_best_match(the_match: Match, group_name: str) -> str: + result = the_match.group(0) + try: + result = the_match.group(group_name) + return result + except IndexError: + pass + try: + result = the_match.group(1) + return result + except IndexError: + pass + return result + match_regexp = r"{{\s*\$match\s*}}" if ( @@ -139,10 +182,9 @@ def get_operation_value( and operation_config[Constants.TAG] ): tag = operation_config[Constants.TAG] - if isinstance(raw_props[operation_key], str): - tag = re.sub( - match_regexp, raw_props[operation_key], tag, 0, re.MULTILINE - ) + tag_id = _get_best_match(match, "tag") + if isinstance(tag_id, str): + tag = re.sub(match_regexp, tag_id, tag, 0, re.MULTILINE) if self.tag_prefix: tag = self.tag_prefix + tag @@ -151,22 +193,32 @@ def get_operation_value( operation_type == Constants.ADD_OWNER_OPERATION and operation_config[Constants.OWNER_TYPE] ): - owner_id = raw_props[operation_key] + owner_id = _get_best_match(match, "owner") + owner_category = ( + operation_config.get(Constants.OWNER_CATEGORY) + or OwnershipTypeClass.DATAOWNER + ) + owner_category = owner_category.upper() if self.strip_owner_email_id: owner_id = self.sanitize_owner_ids(owner_id) if operation_config[Constants.OWNER_TYPE] == Constants.USER_OWNER: - return mce_builder.make_owner_urn(owner_id, OwnerType.USER) + return { + "urn": mce_builder.make_owner_urn(owner_id, OwnerType.USER), + "category": owner_category, + } elif operation_config[Constants.OWNER_TYPE] == Constants.GROUP_OWNER: - return mce_builder.make_owner_urn(owner_id, OwnerType.GROUP) + return { + "urn": mce_builder.make_owner_urn(owner_id, OwnerType.GROUP), + "category": owner_category, + } elif ( operation_type == Constants.ADD_TERM_OPERATION and operation_config[Constants.TERM] ): term = operation_config[Constants.TERM] - if isinstance(raw_props[operation_key], str): - term = re.sub( - match_regexp, raw_props[operation_key], term, 0, re.MULTILINE - ) + captured_term_id = _get_best_match(match, "term") + if isinstance(captured_term_id, str): + term = re.sub(match_regexp, captured_term_id, term, 0, re.MULTILINE) return mce_builder.make_term_urn(term) return None @@ -175,15 +227,13 @@ def sanitize_owner_ids(self, owner_id: str) -> str: owner_id = owner_id[0 : owner_id.index("@")] return owner_id - def is_match(self, match_clause: Any, raw_props_value: Any) -> bool: + def get_match(self, match_clause: Any, raw_props_value: Any) -> Optional[Match]: # function to check if a match clause is satisfied to a value. - is_matching: bool if type(raw_props_value) not in Constants.OPERAND_DATATYPE_SUPPORTED or type( raw_props_value ) != type(match_clause): - is_matching = False + return None elif type(raw_props_value) == str: - is_matching = True if re.match(match_clause, raw_props_value) else False + return re.match(match_clause, raw_props_value) else: - is_matching = match_clause == raw_props_value - return is_matching + return re.match(str(match_clause), str(raw_props_value)) diff --git a/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py b/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py index b5fa3c3a723ea4..ef0db205b72ac9 100644 --- a/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py +++ b/metadata-ingestion/src/datahub/utilities/memory_leak_detector.py @@ -12,7 +12,7 @@ def _trace_has_file(trace: tracemalloc.Traceback, file_pattern: str) -> bool: - for frame_index in range(0, len(trace)): + for frame_index in range(len(trace)): cur_frame = trace[frame_index] if fnmatch.fnmatch(cur_frame.filename, file_pattern): return True @@ -99,8 +99,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: _init_leak_detection() try: - res = func(*args, **kwargs) - return res + return func(*args, **kwargs) finally: if detect_leaks: _perform_leak_detection() diff --git a/contrib/metadata-ingestion/python/.gitkeep b/metadata-ingestion/src/datahub/utilities/registries/__init__.py similarity index 100% rename from contrib/metadata-ingestion/python/.gitkeep rename to metadata-ingestion/src/datahub/utilities/registries/__init__.py diff --git a/metadata-ingestion/src/datahub/utilities/registries/domain_registry.py b/metadata-ingestion/src/datahub/utilities/registries/domain_registry.py new file mode 100644 index 00000000000000..061cfaf2e3b7df --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/registries/domain_registry.py @@ -0,0 +1,50 @@ +import logging +from typing import List, Optional + +from datahub.ingestion.graph.client import DataHubGraph + +logger = logging.getLogger(__name__) + + +class DomainRegistry: + """A class that makes it easy to resolve domains using DataHub""" + + def __init__( + self, + cached_domains: Optional[List[str]] = [], + graph: Optional[DataHubGraph] = None, + ): + self.domain_registry = {} + if cached_domains: + # isolate the domains that don't seem fully specified + domains_needing_resolution = [ + d + for d in cached_domains + if (not d.startswith("urn:li:domain") and d.count("-") != 4) + ] + if domains_needing_resolution and not graph: + raise ValueError( + f"Following domains need server-side resolution {domains_needing_resolution} but a DataHub server wasn't provided. Either use fully qualified domain ids (e.g. urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba) or provide a datahub_api config in your recipe." + ) + for domain_identifier in domains_needing_resolution: + assert graph + # first try to check if this domain exists by urn + maybe_domain_urn = f"urn:li:domain:{domain_identifier}" + maybe_domain_properties = graph.get_domain_properties(maybe_domain_urn) + if maybe_domain_properties: + self.domain_registry[domain_identifier] = maybe_domain_urn + else: + # try to get this domain by name + domain_urn = graph.get_domain_urn_by_name(domain_identifier) + if domain_urn: + self.domain_registry[domain_identifier] = domain_urn + else: + logger.error( + f"Failed to retrieve domain id for domain {domain_identifier}" + ) + raise ValueError( + f"domain {domain_identifier} doesn't seem to be provisioned on DataHub. Either provision it first and re-run ingestion, or provide a fully qualified domain id (e.g. urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba) to skip this check." + ) + + def get_domain_urn(self, domain_identifier: str) -> str: + return self.domain_registry.get(domain_identifier) or domain_identifier diff --git a/metadata-ingestion/src/datahub/utilities/server_config_util.py b/metadata-ingestion/src/datahub/utilities/server_config_util.py index c919a1356f2642..1b8c05b6091347 100644 --- a/metadata-ingestion/src/datahub/utilities/server_config_util.py +++ b/metadata-ingestion/src/datahub/utilities/server_config_util.py @@ -1,17 +1,18 @@ from typing import Any, Dict, Optional -from datahub.telemetry.telemetry import set_telemetry_enable +from datahub.telemetry.telemetry import suppress_telemetry # Only to be written to for logging server related information -global_debug: Dict[str, Any] = dict() +global_debug: Dict[str, Any] = {} def set_gms_config(config: Dict) -> Any: global_debug["gms_config"] = config cli_telemtry_enabled = is_cli_telemetry_enabled() - if cli_telemtry_enabled is not None: - set_telemetry_enable(cli_telemtry_enabled) + if cli_telemtry_enabled is not None and not cli_telemtry_enabled: + # server requires telemetry to be disabled on client + suppress_telemetry() def get_gms_config() -> Dict: diff --git a/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py b/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py index 80ea7cc31455cd..6fe57b297d4528 100644 --- a/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py +++ b/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py @@ -1,3 +1,4 @@ +import contextlib import logging import re import unittest @@ -7,15 +8,12 @@ from sqllineage.core.holders import Column, SQLLineageHolder from sqllineage.exceptions import SQLLineageException -try: +with contextlib.suppress(ImportError): import sqlparse from networkx import DiGraph from sqllineage.core import LineageAnalyzer import datahub.utilities.sqllineage_patch -except ImportError: - pass - logger = logging.getLogger(__name__) @@ -97,7 +95,7 @@ def __init__(self, sql_query: str) -> None: logger.error(f"SQL lineage analyzer error '{e}' for query: '{self._sql}") def get_tables(self) -> List[str]: - result: List[str] = list() + result: List[str] = [] if self._sql_holder is None: logger.error("sql holder not present so cannot get tables") return result @@ -135,12 +133,10 @@ def get_columns(self) -> List[str]: result.add(str(column.raw_name)) # Reverting back all the previously renamed words which confuses the parser - result = set(["date" if c == self._DATE_SWAP_TOKEN else c for c in result]) - result = set( - [ - "timestamp" if c == self._TIMESTAMP_SWAP_TOKEN else c - for c in list(result) - ] - ) + result = {"date" if c == self._DATE_SWAP_TOKEN else c for c in result} + result = { + "timestamp" if c == self._TIMESTAMP_SWAP_TOKEN else c for c in list(result) + } + # swap back renamed date column return list(result) diff --git a/metadata-ingestion/src/datahub/utilities/sql_parser.py b/metadata-ingestion/src/datahub/utilities/sql_parser.py index eb0bc0ec8262f4..28b5082ccbb3b2 100644 --- a/metadata-ingestion/src/datahub/utilities/sql_parser.py +++ b/metadata-ingestion/src/datahub/utilities/sql_parser.py @@ -1,3 +1,4 @@ +import contextlib import logging import multiprocessing import re @@ -9,11 +10,8 @@ from datahub.utilities.sql_lineage_parser_impl import SqlLineageSQLParserImpl -try: +with contextlib.suppress(ImportError): from sql_metadata import Parser as MetadataSQLParser -except ImportError: - pass - logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_query_combiner.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_query_combiner.py index 0474f4ec7d3d68..947f5e30d62c89 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlalchemy_query_combiner.py +++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_query_combiner.py @@ -108,8 +108,7 @@ def get_query_columns(query: Any) -> List[Any]: try: # inner_columns will be more accurate if the column names are unnamed, # since .columns will remove the "duplicates". - inner_columns = list(query.inner_columns) - return inner_columns + return list(query.inner_columns) except AttributeError: return list(query.columns) diff --git a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py index fc10323e72cd9c..760361db21ca40 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py @@ -99,3 +99,20 @@ def _validate_entity_id(entity_id: List[str]) -> None: raise InvalidUrnError( f"Invalid env:{env}. Allowed evn are {DatasetUrn.VALID_FABRIC_SET}" ) + + """A helper function to extract simple . path notation from the v2 field path""" + + @staticmethod + def _get_simple_field_path_from_v2_field_path(field_path: str) -> str: + if field_path.startswith("[version=2.0]"): + # this is a v2 field path + tokens = [ + t + for t in field_path.split(".") + if not (t.startswith("[") or t.endswith("]")) + ] + path = ".".join(tokens) + return path + else: + # not a v2, we assume this is a simple path + return field_path diff --git a/metadata-ingestion/src/datahub/utilities/urns/urn.py b/metadata-ingestion/src/datahub/utilities/urns/urn.py index 7498cc1532c66e..479e74331fd9b3 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/urn.py @@ -21,7 +21,7 @@ class Urn: def __init__( self, entity_type: str, entity_id: List[str], urn_domain: str = LI_DOMAIN ): - if len(entity_id) == 0: + if not entity_id: raise InvalidUrnError("Empty entity id.") self._validate_entity_type(entity_type) self._validate_entity_id(entity_id) @@ -122,9 +122,9 @@ def _get_entity_id_from_str(entity_id: str) -> List[str]: part_start = i + 1 if start_paren_count != 0: - raise InvalidUrnError(f"{entity_id}, mismtached paren nesting") + raise InvalidUrnError(f"{entity_id}, mismatched paren nesting") - parts.append(entity_id[part_start : len(entity_id) - 1]) + parts.append(entity_id[part_start:-1]) return parts @@ -151,11 +151,12 @@ def __hash__(self) -> int: return hash((self._domain, self._entity_type) + tuple(self._entity_id)) def __eq__(self, other: object) -> bool: - if not isinstance(other, Urn): - return False - return ( - self._entity_id == other._entity_id - and self._domain == other._domain - and self._entity_type == other._entity_type + ( + self._entity_id == other._entity_id + and self._domain == other._domain + and self._entity_type == other._entity_type + ) + if isinstance(other, Urn) + else False ) diff --git a/metadata-ingestion/src/datahub_provider/__init__.py b/metadata-ingestion/src/datahub_provider/__init__.py index 05ba1cf1df901b..4c0b2bd8e714ed 100644 --- a/metadata-ingestion/src/datahub_provider/__init__.py +++ b/metadata-ingestion/src/datahub_provider/__init__.py @@ -9,6 +9,16 @@ def get_provider_info(): return { "name": "DataHub", "description": "`DataHub `__\n", + "connection-types": [ + { + "hook-class-name": "datahub_provider.hooks.datahub.DatahubRestHook", + "connection-type": "datahub_rest", + }, + { + "hook-class-name": "datahub_provider.hooks.datahub.DatahubKafkaHook", + "connection-type": "datahub_kafka", + }, + ], "hook-class-names": [ "datahub_provider.hooks.datahub.DatahubRestHook", "datahub_provider.hooks.datahub.DatahubKafkaHook", diff --git a/metadata-ingestion/src/datahub_provider/client/airflow_generator.py b/metadata-ingestion/src/datahub_provider/client/airflow_generator.py index b5c389d298969e..69943df50d3afb 100644 --- a/metadata-ingestion/src/datahub_provider/client/airflow_generator.py +++ b/metadata-ingestion/src/datahub_provider/client/airflow_generator.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Union, cast from airflow.configuration import conf @@ -32,12 +32,11 @@ def _get_dependencies( upstream_task = dag.task_dict[upstream_task_id] # if upstream task is not a subdag, then skip it - if upstream_task.subdag is None: + upstream_subdag = getattr(upstream_task, "subdag", None) + if upstream_subdag is None: continue # else, link the leaf tasks of the upstream subdag as upstream tasks - upstream_subdag = upstream_task.subdag - for upstream_subdag_task_id in upstream_subdag.task_dict: upstream_subdag_task = upstream_subdag.task_dict[ upstream_subdag_task_id @@ -87,15 +86,37 @@ def _get_dependencies( if subdag_task_id in upstream_task._downstream_task_ids: upstream_subdag_triggers.append(upstream_task_urn) + # If the operator is an ExternalTaskSensor then we set the remote task as upstream. + # It is possible to tie an external sensor to DAG if external_task_id is omitted but currently we can't tie + # jobflow to anothet jobflow. + external_task_upstreams = [] + if task.task_type == "ExternalTaskSensor": + from airflow.sensors.external_task_sensor import ExternalTaskSensor + + task = cast(ExternalTaskSensor, task) + if hasattr(task, "external_task_id") and task.external_task_id is not None: + external_task_upstreams = [ + DataJobUrn.create_from_ids( + job_id=task.external_task_id, + data_flow_urn=str( + DataFlowUrn.create_from_ids( + orchestrator=flow_urn.get_orchestrator_name(), + flow_id=task.external_dag_id, + env=flow_urn.get_env(), + ) + ), + ) + ] # exclude subdag operator tasks since these are not emitted, resulting in empty metadata upstream_tasks = ( [ DataJobUrn.create_from_ids(job_id=task_id, data_flow_urn=str(flow_urn)) for task_id in task.upstream_task_ids - if dag.task_dict[task_id].subdag is None + if getattr(dag.task_dict[task_id], "subdag", None) is None ] + upstream_subdag_task_urns + upstream_subdag_triggers + + external_task_upstreams ) return upstream_tasks @@ -114,8 +135,6 @@ def generate_dataflow( :param capture_owner: :return: DataFlow - Data generated dataflow """ - from airflow.serialization.serialized_objects import SerializedDAG - id = dag.dag_id orchestrator = "airflow" description = f"{dag.description}\n\n{dag.doc_md or ''}" @@ -123,13 +142,7 @@ def generate_dataflow( cluster=cluster, id=id, orchestrator=orchestrator, description=description ) - flow_property_bag: Dict[str, str] = { - key: repr(value) - for (key, value) in SerializedDAG.serialize_dag(dag).items() - } - for key in dag.get_serialized_fields(): - if key not in flow_property_bag: - flow_property_bag[key] = repr(getattr(dag, key)) + flow_property_bag: Dict[str, str] = {} allowed_flow_keys = [ "_access_control", @@ -142,9 +155,10 @@ def generate_dataflow( "tags", "timezone", ] - flow_property_bag = { - k: v for (k, v) in flow_property_bag.items() if k in allowed_flow_keys - } + + for key in allowed_flow_keys: + if hasattr(dag, key): + flow_property_bag[key] = repr(getattr(dag, key)) data_flow.properties = flow_property_bag base_url = conf.get("webserver", "base_url") @@ -191,21 +205,13 @@ def generate_datajob( :param capture_tags: bool - whether to set tags automatically from airflow task :return: DataJob - returns the generated DataJob object """ - from airflow.serialization.serialized_objects import SerializedBaseOperator - dataflow_urn = DataFlowUrn.create_from_ids( orchestrator="airflow", env=cluster, flow_id=dag.dag_id ) datajob = DataJob(id=task.task_id, flow_urn=dataflow_urn) datajob.description = AirflowGenerator._get_description(task) - job_property_bag: Dict[str, str] = { - key: repr(value) - for (key, value) in SerializedBaseOperator.serialize_operator(task).items() - } - for key in task.get_serialized_fields(): - if key not in job_property_bag: - job_property_bag[key] = repr(getattr(task, key)) + job_property_bag: Dict[str, str] = {} allowed_task_keys = [ "_downstream_task_ids", @@ -223,9 +229,10 @@ def generate_datajob( "trigger_rule", "wait_for_downstream", ] - job_property_bag = { - k: v for (k, v) in job_property_bag.items() if k in allowed_task_keys - } + + for key in allowed_task_keys: + if hasattr(task, key): + job_property_bag[key] = repr(getattr(task, key)) datajob.properties = job_property_bag base_url = conf.get("webserver", "base_url") diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py new file mode 100644 index 00000000000000..89a037324e7cbe --- /dev/null +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_operator.py @@ -0,0 +1,78 @@ +import datetime +from typing import Any, List, Optional, Sequence, Union + +from airflow.models import BaseOperator + +from datahub.api.circuit_breaker import ( + AssertionCircuitBreaker, + AssertionCircuitBreakerConfig, +) +from datahub_provider.hooks.datahub import DatahubRestHook + + +class DataHubAssertionOperator(BaseOperator): + r""" + DataHub Assertion Circuit Breaker Operator. + + :param urn: The DataHub dataset unique identifier. (templated) + :param datahub_rest_conn_id: The REST datahub connection id to communicate with DataHub + which is set as Airflow connection. + :param check_last_assertion_time: If set it checks assertions after the last operation was set on the dataset. + By default it is True. + :param time_delta: If verify_after_last_update is False it checks for assertion within the time delta. + """ + + template_fields: Sequence[str] = ("urn",) + circuit_breaker: AssertionCircuitBreaker + urn: Union[List[str], str] + + def __init__( # type: ignore[no-untyped-def] + self, + *, + urn: Union[List[str], str], + datahub_rest_conn_id: Optional[str] = None, + check_last_assertion_time: bool = True, + time_delta: Optional[datetime.timedelta] = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + hook: DatahubRestHook + if datahub_rest_conn_id is not None: + hook = DatahubRestHook(datahub_rest_conn_id=datahub_rest_conn_id) + else: + hook = DatahubRestHook() + + host, password, timeout_sec = hook._get_config() + self.urn = urn + config: AssertionCircuitBreakerConfig = AssertionCircuitBreakerConfig( + datahub_host=host, + datahub_token=password, + timeout=timeout_sec, + verify_after_last_update=check_last_assertion_time, + time_delta=time_delta if time_delta else datetime.timedelta(days=1), + ) + + self.circuit_breaker = AssertionCircuitBreaker(config=config) + + def execute(self, context: Any) -> bool: + if "datahub_silence_circuit_breakers" in context["dag_run"].conf: + self.log.info( + "Circuit breaker is silenced because datahub_silence_circuit_breakers config is set" + ) + return True + + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + if type(self.urn) == str: + urns = [self.urn] + elif type(self.urn) == list: + urns = self.urn + else: + raise Exception(f"urn parameter has invalid type {type(self.urn)}") + + for urn in urns: + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + ret = self.circuit_breaker.is_circuit_breaker_active(urn=urn) + if ret: + raise Exception(f"Dataset {self.urn} is not in consumable state") + + return True diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py new file mode 100644 index 00000000000000..55a3492f9c8d64 --- /dev/null +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_assertion_sensor.py @@ -0,0 +1,78 @@ +import datetime +from typing import Any, List, Optional, Sequence, Union + +from airflow.sensors.base import BaseSensorOperator + +from datahub.api.circuit_breaker import ( + AssertionCircuitBreaker, + AssertionCircuitBreakerConfig, +) +from datahub_provider.hooks.datahub import DatahubRestHook + + +class DataHubAssertionSensor(BaseSensorOperator): + r""" + DataHub Assertion Circuit Breaker Sensor. + + :param urn: The DataHub dataset unique identifier. (templated) + :param datahub_rest_conn_id: The REST datahub connection id to communicate with DataHub + which is set as Airflow connection. + :param check_last_assertion_time: If set it checks assertions after the last operation was set on the dataset. + By default it is True. + :param time_delta: If verify_after_last_update is False it checks for assertion within the time delta. + """ + + template_fields: Sequence[str] = ("urn",) + circuit_breaker: AssertionCircuitBreaker + urn: Union[List[str], str] + + def __init__( # type: ignore[no-untyped-def] + self, + *, + urn: Union[List[str], str], + datahub_rest_conn_id: Optional[str] = None, + check_last_assertion_time: bool = True, + time_delta: datetime.timedelta = datetime.timedelta(days=1), + **kwargs, + ) -> None: + super().__init__(**kwargs) + hook: DatahubRestHook + if datahub_rest_conn_id is not None: + hook = DatahubRestHook(datahub_rest_conn_id=datahub_rest_conn_id) + else: + hook = DatahubRestHook() + + host, password, timeout_sec = hook._get_config() + self.urn = urn + config: AssertionCircuitBreakerConfig = AssertionCircuitBreakerConfig( + datahub_host=host, + datahub_token=password, + timeout=timeout_sec, + verify_after_last_update=check_last_assertion_time, + time_delta=time_delta, + ) + self.circuit_breaker = AssertionCircuitBreaker(config=config) + + def poke(self, context: Any) -> bool: + if "datahub_silence_circuit_breakers" in context["dag_run"].conf: + self.log.info( + "Circuit breaker is silenced because datahub_silence_circuit_breakers config is set" + ) + return True + + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + if type(self.urn) == str: + urns = [self.urn] + elif type(self.urn) == list: + urns = self.urn + else: + raise Exception(f"urn parameter has invalid type {type(self.urn)}") + + for urn in urns: + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + ret = self.circuit_breaker.is_circuit_breaker_active(urn=urn) + if ret: + self.log.info(f"Dataset {self.urn} is not in consumable state") + return False + + return True diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py new file mode 100644 index 00000000000000..905b7f71e23ca5 --- /dev/null +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_operator.py @@ -0,0 +1,98 @@ +import datetime +from typing import Any, List, Optional, Sequence, Union + +from airflow.sensors.base import BaseSensorOperator + +from datahub.api.circuit_breaker import ( + OperationCircuitBreaker, + OperationCircuitBreakerConfig, +) +from datahub_provider.hooks.datahub import DatahubRestHook + + +class DataHubOperationCircuitBreakerOperator(BaseSensorOperator): + r""" + DataHub Operation Circuit Breaker Operator. + + :param urn: The DataHub dataset unique identifier. (templated) + :param datahub_rest_conn_id: The REST datahub connection id to communicate with DataHub + which is set as Airflow connection. + :param partition: The partition to check the operation. + :param source_type: The partition to check the operation. :ref:`https://datahubproject.io/docs/graphql/enums#operationsourcetype` + + """ + + template_fields: Sequence[str] = ( + "urn", + "partition", + "source_type", + "operation_type", + ) + circuit_breaker: OperationCircuitBreaker + urn: Union[List[str], str] + partition: Optional[str] + source_type: Optional[str] + operation_type: Optional[str] + + def __init__( # type: ignore[no-untyped-def] + self, + *, + urn: Union[List[str], str], + datahub_rest_conn_id: Optional[str] = None, + time_delta: Optional[datetime.timedelta] = datetime.timedelta(days=1), + partition: Optional[str] = None, + source_type: Optional[str] = None, + operation_type: Optional[str] = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + hook: DatahubRestHook + if datahub_rest_conn_id is not None: + hook = DatahubRestHook(datahub_rest_conn_id=datahub_rest_conn_id) + else: + hook = DatahubRestHook() + + host, password, timeout_sec = hook._get_config() + + self.urn = urn + self.partition = partition + self.operation_type = operation_type + self.source_type = source_type + + config: OperationCircuitBreakerConfig = OperationCircuitBreakerConfig( + datahub_host=host, + datahub_token=password, + timeout=timeout_sec, + time_delta=time_delta, + ) + + self.circuit_breaker = OperationCircuitBreaker(config=config) + + def execute(self, context: Any) -> bool: + if "datahub_silence_circuit_breakers" in context["dag_run"].conf: + self.log.info( + "Circuit breaker is silenced because datahub_silence_circuit_breakers config is set" + ) + return True + + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + if type(self.urn) == str: + urns = [self.urn] + elif type(self.urn) == list: + urns = self.urn + else: + raise Exception(f"urn parameter has invalid type {type(self.urn)}") + + partition: Optional[str] + for urn in urns: + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + ret = self.circuit_breaker.is_circuit_breaker_active( + urn=urn, + partition=self.partition, + operation_type=self.operation_type, + source_type=self.source_type, + ) + if ret: + raise Exception(f"Dataset {self.urn} is not in consumable state") + + return True diff --git a/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py new file mode 100644 index 00000000000000..31b387a7e65b05 --- /dev/null +++ b/metadata-ingestion/src/datahub_provider/operators/datahub_operation_sensor.py @@ -0,0 +1,100 @@ +import datetime +from typing import Any, List, Optional, Sequence, Union + +from airflow.sensors.base import BaseSensorOperator + +from datahub.api.circuit_breaker import ( + OperationCircuitBreaker, + OperationCircuitBreakerConfig, +) +from datahub_provider.hooks.datahub import DatahubRestHook + + +class DataHubOperationCircuitBreakerSensor(BaseSensorOperator): + r""" + DataHub Operation Circuit Breaker Sensor. + + :param urn: The DataHub dataset unique identifier. (templated) + :param datahub_rest_conn_id: The REST datahub connection id to communicate with DataHub + which is set as Airflow connection. + :param partition: The partition to check the operation. + :param source_type: The source type to filter on. If not set it will accept any source type. + See valid values at: https://datahubproject.io/docs/graphql/enums#operationsourcetype + :param operation_type: The operation type to filter on. If not set it will accept any source type. + See valid values at: https://datahubproject.io/docs/graphql/enums/#operationtype + """ + + template_fields: Sequence[str] = ( + "urn", + "partition", + "source_type", + "operation_type", + ) + circuit_breaker: OperationCircuitBreaker + urn: Union[List[str], str] + partition: Optional[str] + source_type: Optional[str] + operation_type: Optional[str] + + def __init__( # type: ignore[no-untyped-def] + self, + *, + urn: Union[List[str], str], + datahub_rest_conn_id: Optional[str] = None, + time_delta: Optional[datetime.timedelta] = datetime.timedelta(days=1), + partition: Optional[str] = None, + source_type: Optional[str] = None, + operation_type: Optional[str] = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + hook: DatahubRestHook + if datahub_rest_conn_id is not None: + hook = DatahubRestHook(datahub_rest_conn_id=datahub_rest_conn_id) + else: + hook = DatahubRestHook() + + host, password, timeout_sec = hook._get_config() + + self.urn = urn + self.partition = partition + self.operation_type = operation_type + self.source_type = source_type + + config: OperationCircuitBreakerConfig = OperationCircuitBreakerConfig( + datahub_host=host, + datahub_token=password, + timeout=timeout_sec, + time_delta=time_delta, + ) + + self.circuit_breaker = OperationCircuitBreaker(config=config) + + def poke(self, context: Any) -> bool: + if "datahub_silence_circuit_breakers" in context["dag_run"].conf: + self.log.info( + "Circuit breaker is silenced because datahub_silence_circuit_breakers config is set" + ) + return True + + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + if type(self.urn) == str: + urns = [self.urn] + elif type(self.urn) == list: + urns = self.urn + else: + raise Exception(f"urn parameter has invalid type {type(self.urn)}") + + for urn in urns: + self.log.info(f"Checking if dataset {self.urn} is ready to be consumed") + ret = self.circuit_breaker.is_circuit_breaker_active( + urn=urn, + partition=self.partition, + operation_type=self.operation_type, + source_type=self.source_type, + ) + if ret: + self.log.info(f"Dataset {self.urn} is not in consumable state") + return False + + return True diff --git a/metadata-ingestion/tests/conftest.py b/metadata-ingestion/tests/conftest.py index 800232c921c640..0390547800b765 100644 --- a/metadata-ingestion/tests/conftest.py +++ b/metadata-ingestion/tests/conftest.py @@ -4,8 +4,21 @@ import pytest -from tests.test_helpers.docker_helpers import docker_compose_runner # noqa: F401 -from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401 +# Enable debug logging. +logging.getLogger().setLevel(logging.DEBUG) +os.environ["DATAHUB_DEBUG"] = "1" + +# Disable telemetry +os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false" + +# Reduce retries on GMS, because this causes tests to hang while sleeping +# between retries. +os.environ["DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES"] = "1" + +# We need our imports to go below the os.environ updates, since mere act +# of importing some datahub modules will load env variables. +from tests.test_helpers.docker_helpers import docker_compose_runner # noqa: F401,E402 +from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401,E402 try: # See https://github.com/spulec/freezegun/issues/98#issuecomment-590553475. @@ -13,13 +26,6 @@ except ImportError: pass -# Enable debug logging. -logging.getLogger().setLevel(logging.DEBUG) -os.putenv("DATAHUB_DEBUG", "1") - -# Disable telemetry -os.putenv("DATAHUB_TELEMETRY_ENABLED", "false") - @pytest.fixture def mock_time(monkeypatch): diff --git a/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_default_config.json b/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_default_config.json index 9bb76f1457b708..48d1838b1b9c2c 100644 --- a/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_default_config.json +++ b/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_default_config.json @@ -22,6 +22,8 @@ "systemMetadata": { "lastObserved": 1629795600000, "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -48,6 +50,27 @@ "systemMetadata": { "lastObserved": 1629795600000, "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:groupDisplayName2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -87,6 +110,27 @@ "systemMetadata": { "lastObserved": 1629795600000, "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:johngreen@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -126,6 +170,27 @@ "systemMetadata": { "lastObserved": 1629795600000, "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:adamhall@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, "properties": null } } diff --git a/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_nested_groups.json b/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_nested_groups.json index 0f9d738b1ca932..6c1cb3e8e52beb 100644 --- a/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_nested_groups.json +++ b/metadata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_nested_groups.json @@ -1,150 +1,344 @@ [ - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { - "urn": "urn:li:corpGroup:nestedgroupDisplayName1", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "displayName": "nestedgroupDisplayName1", - "email": null, - "admins": [], - "members": [], - "groups": [], - "description": "Placeholder group with members from nested groups." - } +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:nestedgroupDisplayName1", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "displayName": "nestedgroupDisplayName1", + "email": null, + "admins": [], + "members": [], + "groups": [], + "description": "Placeholder group with members from nested groups." } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1629795600000, - "runId": "test-azure-ad", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { - "urn": "urn:li:corpuser:foobar@acryl.io", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpUserInfo": { - "active": true, - "displayName": "Foo Bar", - "email": "foobar@acryl.io", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, - "firstName": "Foo", - "lastName": "Bar", - "fullName": "Foo Bar", - "countryCode": null - } - }, - { - "com.linkedin.pegasus2avro.identity.GroupMembership": { - "groups": [ - "urn:li:corpGroup:nestedgroupDisplayName1" - ] - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:nestedgroupDisplayName1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:foobar@acryl.io", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Foo Bar", + "email": "foobar@acryl.io", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Foo", + "lastName": "Bar", + "fullName": "Foo Bar", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:nestedgroupDisplayName1" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1629795600000, - "runId": "test-azure-ad", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { - "urn": "urn:li:corpuser:johngreen@acryl.io", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpUserInfo": { - "active": true, - "displayName": "John Green", - "email": "johngreen@acryl.io", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, - "firstName": "John", - "lastName": "Green", - "fullName": "John Green", - "countryCode": null - } - }, - { - "com.linkedin.pegasus2avro.identity.GroupMembership": { - "groups": [ - "urn:li:corpGroup:nestedgroupDisplayName1" - ] - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:foobar@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:johngreen@acryl.io", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "John Green", + "email": "johngreen@acryl.io", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "John", + "lastName": "Green", + "fullName": "John Green", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:nestedgroupDisplayName1" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1629795600000, - "runId": "test-azure-ad", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { - "urn": "urn:li:corpuser:adamhall@acryl.io", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpUserInfo": { - "active": true, - "displayName": "Adam Hall", - "email": "adamhall@acryl.io", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, - "firstName": "Adam", - "lastName": "Hall", - "fullName": "Adam Hall", - "countryCode": null - } - }, - { - "com.linkedin.pegasus2avro.identity.GroupMembership": { - "groups": [ - "urn:li:corpGroup:nestedgroupDisplayName1" - ] - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:johngreen@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:adamhall@acryl.io", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Adam Hall", + "email": "adamhall@acryl.io", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Adam", + "lastName": "Hall", + "fullName": "Adam Hall", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:nestedgroupDisplayName1" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1629795600000, - "runId": "test-azure-ad", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:adamhall@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:johngreen@acryl.io", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "John Green", + "email": "johngreen@acryl.io", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "John", + "lastName": "Green", + "fullName": "John Green", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:nestedgroupDisplayName1" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:johngreen@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:adamhall@acryl.io", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Adam Hall", + "email": "adamhall@acryl.io", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Adam", + "lastName": "Hall", + "fullName": "Adam Hall", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:nestedgroupDisplayName1" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:adamhall@acryl.io", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1629795600000, + "runId": "test-azure-ad", + "registryName": null, + "registryVersion": null, + "properties": null } +} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_with_read_events_golden.json b/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_with_read_events_golden.json new file mode 100644 index 00000000000000..ab240457410e39 --- /dev/null +++ b/metadata-ingestion/tests/integration/bigquery-usage/bigquery_usages_with_read_events_golden.json @@ -0,0 +1,1313 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1593\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_48d74e9b_179ab204cca\", \"text\": \"# SELECT * FROM `bigquery-public-data.austin_311.311_service_requests` LIMIT 10\\n\\n#CREATE SCHEMA IF NOT EXISTS `test_schema`\\n\\nCREATE OR REPLACE TABLE test_schema.austin311_derived AS (SELECT * FROM `bigquery-public-data.austin_311.311_service_requests` LIMIT 10)\", \"bytesProcessed\": \"315621376\"}, \"lastUpdatedTimestamp\": 1622073693816}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"unique_key,complaint_type,complaint_description,owning_department,source,status,status_change_date,created_date,last_update_date,close_date,incident_address,street_number,street_name,city,incident_zip,county,state_plane_x_coordinate,state_plane_y_coordinate,latitude,longitude,location,council_district_code,map_page,map_tile\"}, \"lastUpdatedTimestamp\": 1622073700622}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.location,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622073731199}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622073734136}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1544\", \"sessionId\": \"projects/harshal-playground-306419/jobs/script_job_0f7e44a1fc2936fcdee8d54ad6980563_1\", \"text\": \"CREATE OR REPLACE TABLE test_schema.austin311_derived AS (SELECT * FROM `bigquery-public-data.austin_311.311_service_requests` LIMIT 10)\", \"bytesProcessed\": \"315621376\"}, \"lastUpdatedTimestamp\": 1622074056868}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622074109963}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622075542713}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622075551406}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.austin_311.311_service_requests,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"17160\", \"sessionId\": \"projects/harshal-playground-306419/jobs/script_job_60b5a82671e101a039802bf0f6577257_1\", \"text\": \"CREATE OR REPLACE TABLE test_schema.austin311_derived AS (SELECT * FROM `bigquery-public-data.austin_311.311_service_requests` LIMIT 10)\", \"bytesProcessed\": \"315621376\"}, \"lastUpdatedTimestamp\": 1622075993220}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"308\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_2e6e1b58_179ab44aa4c\", \"text\": \"SELECT * FROM `harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity_20210527` LIMIT 1000\", \"readReason\": \"JOB\", \"fieldsRead\": \"traceSampled,operation.last,protopayload_auditlog.metadataJson,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.protocol,httpRequest.cacheLookup,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.numResponseItems,httpRequest.referer,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.permission,httpRequest.cacheFillBytes,protopayload_auditlog.requestMetadata.requestAttributes.auth,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.requestMetadata,receiveTimestamp,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal,protopayload_auditlog.methodName,protopayload_auditlog.requestMetadata.requestAttributes.reason,resource.labels,protopayload_auditlog.authorizationInfo.resourceAttributes.name,logName,httpRequest.cacheHit,protopayload_auditlog.authenticationInfo,operation,protopayload_auditlog.status.code,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.requestMetadata.requestAttributes.headers,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.requestMetadata.destinationAttributes,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.status,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,httpRequest.responseSize,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resourceAttributes.labels,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.requestMetadata.destinationAttributes.labels,operation.producer,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,spanId,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.destinationAttributes.port,httpRequest.protocol,protopayload_auditlog.status.message,httpRequest.serverIp,sourceLocation.function,httpRequest.requestMethod,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,httpRequest.status,httpRequest.userAgent,protopayload_auditlog.authenticationInfo.principalEmail,httpRequest.cacheValidatedWithOriginServer,operation.first,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.resourceLocation.originalLocations,sourceLocation,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,severity,protopayload_auditlog.requestMetadata.callerIp,httpRequest,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations,resource.type,trace,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes,protopayload_auditlog.resourceLocation,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.serviceName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,sourceLocation.line,insertId,textPayload,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,httpRequest.requestUrl,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,resource.labels.dataset_id,protopayload_auditlog.resourceName,operation.id,protopayload_auditlog.authenticationInfo.authoritySelector,timestamp,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,resource.labels.project_id,protopayload_auditlog.requestMetadata.requestAttributes,httpRequest.requestSize,protopayload_auditlog,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.method,httpRequest.remoteIp,resource,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,sourceLocation.file,protopayload_auditlog.requestMetadata.destinationAttributes.principal\"}, \"lastUpdatedTimestamp\": 1622076075340}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1386\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_376243d3_179ab45d7cd\", \"text\": \"# SELECT * FROM `bigquery-public-data.austin_311.311_service_requests` LIMIT 10\\n\\n# CREATE SCHEMA IF NOT EXISTS `test_schema`;\\n\\nCREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10)\"}, \"lastUpdatedTimestamp\": 1622076153701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1478\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_47cd7ffe_179ab51c580\", \"text\": \"CREATE OR REPLACE TABLE\\n test_schema.excess_deaths_derived AS (\\n SELECT\\n *\\n FROM\\n `bigquery-public-data.covid19_nyt.excess_deaths`\\n LIMIT\\n 10)\"}, \"lastUpdatedTimestamp\": 1622076935475}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.location,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622089936712}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"305\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_5bca1df2_179ac194609\", \"text\": \"SELECT * FROM `harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access_20210526`\", \"readReason\": \"JOB\", \"fieldsRead\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.requestMetadata.requestAttributes.query,resource.labels.project_id,httpRequest.cacheValidatedWithOriginServer,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,operation.first,operation.id,protopayload_auditlog.requestMetadata.requestAttributes.time,httpRequest.requestSize,protopayload_auditlog.authorizationInfo.resourceAttributes.service,insertId,protopayload_auditlog.authorizationInfo.resourceAttributes,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,operation,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal,resource.type,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog,httpRequest.remoteIp,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.numResponseItems,resource.labels,protopayload_auditlog.metadataJson,timestamp,sourceLocation,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.status.code,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.resourceLocation.originalLocations,resource.labels.location,httpRequest.requestMethod,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.destinationAttributes.labels,protopayload_auditlog.status,httpRequest.referer,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,logName,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,receiveTimestamp,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations,textPayload,traceSampled,protopayload_auditlog.resourceLocation,protopayload_auditlog.requestMetadata.destinationAttributes.principal,operation.producer,resource,httpRequest.cacheFillBytes,httpRequest.responseSize,protopayload_auditlog.requestMetadata.requestAttributes,protopayload_auditlog.requestMetadata.requestAttributes.protocol,httpRequest.serverIp,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.serviceName,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,httpRequest.cacheHit,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.authenticationInfo,sourceLocation.file,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,httpRequest.protocol,protopayload_auditlog.requestMetadata,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.status.message,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,sourceLocation.function,httpRequest.requestUrl,httpRequest.cacheLookup,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,httpRequest.userAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,severity,httpRequest,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.destinationAttributes,operation.last,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.auth,protopayload_auditlog.requestMetadata.requestAttributes.headers,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.authorizationInfo.resourceAttributes.labels,httpRequest.status,protopayload_auditlog.methodName,protopayload_auditlog.requestMetadata.requestAttributes.path,sourceLocation.line,trace,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,spanId,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.requestMetadata.destinationAttributes.ip\"}, \"lastUpdatedTimestamp\": 1622090009041}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"253\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_1fcb7394_179ac19894a\", \"text\": \"SELECT * FROM `harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access_20210526`\", \"readReason\": \"JOB\", \"fieldsRead\": \"operation,httpRequest.cacheFillBytes,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.requestMetadata.requestAttributes.host,operation.id,protopayload_auditlog.requestMetadata.requestAttributes.auth,logName,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog,receiveTimestamp,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo,operation.producer,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.method,sourceLocation.function,protopayload_auditlog.requestMetadata.requestAttributes.headers,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,httpRequest.userAgent,httpRequest.responseSize,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,trace,protopayload_auditlog.resourceLocation,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.status.message,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.resourceName,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.authorizationInfo.permission,httpRequest.requestSize,protopayload_auditlog.authenticationInfo.principalSubject,httpRequest.referer,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.authenticationInfo,resource.type,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.metadataJson,httpRequest.cacheHit,operation.last,protopayload_auditlog.authorizationInfo.resourceAttributes,timestamp,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.requestMetadata.requestAttributes.scheme,operation.first,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,traceSampled,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.numResponseItems,resource.labels.location,resource,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.resourceLocation.originalLocations,resource.labels,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.authorizationInfo.resourceAttributes.labels,protopayload_auditlog.methodName,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,sourceLocation,httpRequest.cacheLookup,httpRequest,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.resourceLocation.currentLocations,httpRequest.requestMethod,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.requestMetadata,protopayload_auditlog.status.code,protopayload_auditlog.authorizationInfo,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,resource.labels.project_id,httpRequest.serverIp,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,insertId,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,httpRequest.cacheValidatedWithOriginServer,sourceLocation.file,sourceLocation.line,protopayload_auditlog.status,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,severity,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,httpRequest.protocol,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes,httpRequest.status,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,httpRequest.remoteIp,protopayload_auditlog.requestMetadata.destinationAttributes.port,httpRequest.requestUrl,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.serviceName,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.destinationAttributes,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,spanId,textPayload,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.labels\"}, \"lastUpdatedTimestamp\": 1622090025969}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"428\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_2b9360d9_179ac2f13a6\", \"text\": \"SELECT * FROM `harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access_20210526`\", \"readReason\": \"JOB\", \"fieldsRead\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,sourceLocation.file,httpRequest.responseSize,receiveTimestamp,protopayload_auditlog.methodName,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,resource.labels,httpRequest.cacheFillBytes,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.status,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,httpRequest.referer,protopayload_auditlog.requestMetadata.requestAttributes.id,trace,operation.last,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.status.message,httpRequest.requestUrl,timestamp,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.requestMetadata.requestAttributes.time,resource,protopayload_auditlog.requestMetadata.destinationAttributes.ip,spanId,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,operation.first,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.labels,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.serviceName,resource.labels.project_id,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.authorizationInfo.permission,severity,httpRequest.serverIp,protopayload_auditlog.requestMetadata.requestAttributes,protopayload_auditlog.authorizationInfo,traceSampled,httpRequest.status,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.requestMetadata.requestAttributes.protocol,operation,protopayload_auditlog.requestMetadata.destinationAttributes.labels,protopayload_auditlog.requestMetadata.destinationAttributes.principal,logName,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authenticationInfo.authoritySelector,httpRequest.protocol,resource.type,resource.labels.location,operation.id,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.resourceName,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,httpRequest.cacheHit,protopayload_auditlog.resourceLocation.currentLocations,sourceLocation.function,protopayload_auditlog.requestMetadata.requestAttributes.size,httpRequest.userAgent,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.requestAttributes.auth,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.destinationAttributes,httpRequest.requestSize,protopayload_auditlog.authorizationInfo.resourceAttributes,sourceLocation,protopayload_auditlog.metadataJson,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.requestMetadata,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.resourceLocation,httpRequest.cacheValidatedWithOriginServer,protopayload_auditlog.authenticationInfo,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo,sourceLocation.line,httpRequest,httpRequest.remoteIp,insertId,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.status.code,operation.producer,protopayload_auditlog.requestMetadata.requestAttributes.query,httpRequest.cacheLookup,protopayload_auditlog.requestMetadata.requestAttributes.headers,protopayload_auditlog.authorizationInfo.granted,httpRequest.requestMethod,textPayload,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.numResponseItems\"}, \"lastUpdatedTimestamp\": 1622091438070}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1318\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_6dea4de6_179ac2f4a1a\", \"text\": \"\\n\\nCREATE OR REPLACE TABLE\\n test_schema.excess_deaths_derived AS (\\n SELECT\\n *\\n FROM\\n `bigquery-public-data.covid19_nyt.excess_deaths`\\n LIMIT\\n 10)\\n\"}, \"lastUpdatedTimestamp\": 1622091452779}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"298\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_4ec8791f_179ac3c8bd7\", \"text\": \"SELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"placename,excess_deaths,deaths,end_date,frequency,expected_deaths,start_date,baseline,year,month,week,country\"}, \"lastUpdatedTimestamp\": 1622092320506}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.name,resource.labels.destination,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.requestJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622097989844}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622097993482}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,resource.labels.location,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622097997228}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.location,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622097999488}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.name,resource.labels.destination,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.requestJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098001818}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098023694}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.location,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098042698}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,resource.labels.location,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098053547}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.name,resource.labels.destination,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.requestJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098055160}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098064754}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098069153}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098097118}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098098167}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098188750}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1354\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_26a374d7_179ac970d56\", \"text\": \"CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\"}, \"lastUpdatedTimestamp\": 1622098252897}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"275\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_da0b277_179ac972193\", \"text\": \"\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\\n\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"frequency,deaths,country,expected_deaths,placename,week,excess_deaths,end_date,baseline,month,start_date,year\"}, \"lastUpdatedTimestamp\": 1622098256930}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622098341462}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622145591988}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1627\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_351fc8d9_179af91221c\", \"text\": \"CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\"}, \"lastUpdatedTimestamp\": 1622148197222}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"215\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_4199db62_179af913b72\", \"text\": \"\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\\n\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"frequency,expected_deaths,month,start_date,excess_deaths,year,placename,week,baseline,country,deaths,end_date\"}, \"lastUpdatedTimestamp\": 1622148201980}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1195\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_173540b9_179af9957ac\", \"text\": \"CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\"}, \"lastUpdatedTimestamp\": 1622148734552}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"293\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_315880a1_179af996329\", \"text\": \"\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\\n\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"end_date,week,placename,baseline,frequency,excess_deaths,expected_deaths,country,month,deaths,start_date,year\"}, \"lastUpdatedTimestamp\": 1622148736668}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622148763773}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622148767952}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"288\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_411f77e_179b0306905\", \"text\": \"SELECT * FROM `harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity` WHERE DATE(timestamp) = \\\"2021-05-27\\\" LIMIT 1000\", \"readReason\": \"JOB\", \"fieldsRead\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,spanId,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,httpRequest.responseSize,protopayload_auditlog.methodName,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.authorizationInfo.resourceAttributes.labels,protopayload_auditlog.requestMetadata.destinationAttributes,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,trace,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.status.message,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,sourceLocation,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo,protopayload_auditlog.authorizationInfo.permission,timestamp,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,httpRequest.cacheFillBytes,operation,protopayload_auditlog.resourceLocation,operation.producer,severity,httpRequest.status,protopayload_auditlog.authorizationInfo.granted,receiveTimestamp,protopayload_auditlog.requestMetadata.requestAttributes.headers,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,resource.type,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations,protopayload_auditlog.requestMetadata.callerNetwork,httpRequest.referer,protopayload_auditlog.resourceLocation.originalLocations,httpRequest.remoteIp,protopayload_auditlog.authenticationInfo,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.destinationAttributes.labels,protopayload_auditlog.requestMetadata.destinationAttributes.port,textPayload,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,operation.first,sourceLocation.function,protopayload_auditlog.resourceName,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,resource.labels,httpRequest.requestUrl,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.requestMetadata.requestAttributes.id,httpRequest.serverIp,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.requestMetadata.requestAttributes.protocol,sourceLocation.file,httpRequest,protopayload_auditlog.metadataJson,protopayload_auditlog.status,protopayload_auditlog.status.code,sourceLocation.line,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,httpRequest.requestMethod,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.auth,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.serviceName,protopayload_auditlog.requestMetadata,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo,resource.labels.project_id,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.requestMetadata.requestAttributes.reason,httpRequest.cacheValidatedWithOriginServer,resource.labels.dataset_id,protopayload_auditlog,traceSampled,protopayload_auditlog.requestMetadata.requestAttributes.time,httpRequest.userAgent,resource,httpRequest.protocol,httpRequest.requestSize,httpRequest.cacheLookup,protopayload_auditlog.requestMetadata.requestAttributes,protopayload_auditlog.numResponseItems,operation.id,httpRequest.cacheHit,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.authorizationInfo.resourceAttributes,logName,insertId,operation.last,protopayload_auditlog.requestMetadata.destinationAttributes.principal\"}, \"lastUpdatedTimestamp\": 1622158634042}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"612\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_4eb90757_179b030a374\", \"text\": \"SELECT * FROM `harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access` WHERE DATE(timestamp) = \\\"2021-05-27\\\" LIMIT 1000\", \"readReason\": \"JOB\", \"fieldsRead\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role,receiveTimestamp,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.resourceName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews,protopayload_auditlog.requestMetadata,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,protopayload_auditlog,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId,httpRequest.requestMethod,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed,httpRequest.protocol,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression,protopayload_auditlog.serviceName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource,operation.last,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition,sourceLocation.line,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse,protopayload_auditlog.authorizationInfo,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition,resource.labels,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId,httpRequest.requestSize,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,resource.labels.project_id,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.requestMetadata.destinationAttributes.labels,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed,sourceLocation,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,logName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest,httpRequest,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.policyResponse,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.status.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName,protopayload_auditlog.requestMetadata.requestAttributes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId,sourceLocation.file,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents,spanId,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset,httpRequest.userAgent,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType,httpRequest.referer,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset,timestamp,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName,protopayload_auditlog.requestMetadata.requestAttributes.headers,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location,protopayload_auditlog.authenticationInfo,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition,httpRequest.serverIp,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members,operation.id,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName,httpRequest.requestUrl,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime,protopayload_auditlog.methodName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime,operation.producer,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId,severity,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query,httpRequest.status,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.status.code,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables,protopayload_auditlog.authorizationInfo.resourceAttributes,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,httpRequest.cacheHit,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables,protopayload_auditlog.requestMetadata.requestAttributes.auth,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value,insertId,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount,sourceLocation.function,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable,protopayload_auditlog.numResponseItems,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state,protopayload_auditlog.requestMetadata.callerIp,httpRequest.cacheLookup,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load,protopayload_auditlog.resourceLocation,textPayload,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition,trace,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables,resource.type,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType,httpRequest.responseSize,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,operation,operation.first,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract,traceSampled,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse,protopayload_auditlog.authorizationInfo.resourceAttributes.type,httpRequest.cacheValidatedWithOriginServer,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,resource,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy,protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract,httpRequest.cacheFillBytes,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId,httpRequest.remoteIp,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message,protopayload_auditlog.status,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource,protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority,protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId,protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code,protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId,protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition,protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role,protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition,protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId,protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime,protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId\"}, \"lastUpdatedTimestamp\": 1622158649404}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1622160978458}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1888\", \"sessionId\": \"projects/harshal-playground-306419/jobs/script_job_25aa36764d9789d10fb1b4ca06a02245_0\", \"text\": \"CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10)\"}, \"lastUpdatedTimestamp\": 1622161940000}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"229\", \"sessionId\": \"projects/harshal-playground-306419/jobs/script_job_4850ef79fcf5804d113884c9e884b93f_1\", \"text\": \"SELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"deaths,end_date,excess_deaths,frequency,placename,start_date,week,baseline,country,month,expected_deaths,year\"}, \"lastUpdatedTimestamp\": 1622161940571}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419._d60e97aec7f471046a960419adb6d44e98300db7.anon4144b5ed124b4996e1706fa2b1c2a5f82f3d3356,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SCRIPT\", \"affectedDatasets\": [], \"customProperties\": {\"millisecondsTaken\": \"3313\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_a7cc4db_179b062d2d2\", \"text\": \"CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\", \"bytesProcessed\": \"10485760\"}, \"lastUpdatedTimestamp\": 1622161940855}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CREATE\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_nyt.excess_deaths,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1011\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_17c46210_179b543a01a\", \"text\": \"CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\n# SELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\"}, \"lastUpdatedTimestamp\": 1622243780153}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"295\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_7daab8ea_179b54530d8\", \"text\": \"# CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"end_date,excess_deaths,frequency,week,month,year,baseline,start_date,placename,deaths,expected_deaths,country\"}, \"lastUpdatedTimestamp\": 1622243882147}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"342\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_1a73932c_179c8d4433f\", \"text\": \"# CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"excess_deaths,frequency,deaths,baseline,week,month,placename,country,start_date,year,expected_deaths,end_date\"}, \"lastUpdatedTimestamp\": 1622572025571}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"230\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_301191d0_179c8d4c23a\", \"text\": \"# CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.austin311_derived`;\\n\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"last_update_date,created_date,location,city,complaint_type,status_change_date,street_number,state_plane_x_coordinate,incident_zip,source,council_district_code,map_tile,state_plane_y_coordinate,map_page,unique_key,owning_department,street_name,complaint_description,longitude,incident_address,status,close_date,latitude,county\"}, \"lastUpdatedTimestamp\": 1622572057918}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"TABLEDATA_LIST_REQUEST\", \"fieldsRead\": \"logName,resource.type,resource.labels.project_id,resource.labels.dataset_id,protopayload_auditlog.serviceName,protopayload_auditlog.methodName,protopayload_auditlog.resourceName,protopayload_auditlog.resourceLocation.currentLocations,protopayload_auditlog.resourceLocation.originalLocations,protopayload_auditlog.numResponseItems,protopayload_auditlog.status.code,protopayload_auditlog.status.message,protopayload_auditlog.authenticationInfo.principalEmail,protopayload_auditlog.authenticationInfo.authoritySelector,protopayload_auditlog.authenticationInfo.serviceAccountKeyName,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject,protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail,protopayload_auditlog.authenticationInfo.principalSubject,protopayload_auditlog.authorizationInfo.resource,protopayload_auditlog.authorizationInfo.permission,protopayload_auditlog.authorizationInfo.granted,protopayload_auditlog.authorizationInfo.resourceAttributes.service,protopayload_auditlog.authorizationInfo.resourceAttributes.name,protopayload_auditlog.authorizationInfo.resourceAttributes.type,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key,protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value,protopayload_auditlog.authorizationInfo.resourceAttributes.uid,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key,protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value,protopayload_auditlog.authorizationInfo.resourceAttributes.displayName,protopayload_auditlog.authorizationInfo.resourceAttributes.createTime,protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime,protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime,protopayload_auditlog.authorizationInfo.resourceAttributes.etag,protopayload_auditlog.authorizationInfo.resourceAttributes.location,protopayload_auditlog.requestMetadata.callerIp,protopayload_auditlog.requestMetadata.callerSuppliedUserAgent,protopayload_auditlog.requestMetadata.callerNetwork,protopayload_auditlog.requestMetadata.requestAttributes.id,protopayload_auditlog.requestMetadata.requestAttributes.method,protopayload_auditlog.requestMetadata.requestAttributes.headers.key,protopayload_auditlog.requestMetadata.requestAttributes.headers.value,protopayload_auditlog.requestMetadata.requestAttributes.path,protopayload_auditlog.requestMetadata.requestAttributes.host,protopayload_auditlog.requestMetadata.requestAttributes.scheme,protopayload_auditlog.requestMetadata.requestAttributes.query,protopayload_auditlog.requestMetadata.requestAttributes.time,protopayload_auditlog.requestMetadata.requestAttributes.size,protopayload_auditlog.requestMetadata.requestAttributes.protocol,protopayload_auditlog.requestMetadata.requestAttributes.reason,protopayload_auditlog.requestMetadata.requestAttributes.auth.principal,protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences,protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter,protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels,protopayload_auditlog.requestMetadata.destinationAttributes.ip,protopayload_auditlog.requestMetadata.destinationAttributes.port,protopayload_auditlog.requestMetadata.destinationAttributes.labels.key,protopayload_auditlog.requestMetadata.destinationAttributes.labels.value,protopayload_auditlog.requestMetadata.destinationAttributes.principal,protopayload_auditlog.requestMetadata.destinationAttributes.regionCode,protopayload_auditlog.metadataJson,textPayload,timestamp,receiveTimestamp,severity,insertId,httpRequest.requestMethod,httpRequest.requestUrl,httpRequest.requestSize,httpRequest.status,httpRequest.responseSize,httpRequest.userAgent,httpRequest.remoteIp,httpRequest.serverIp,httpRequest.referer,httpRequest.cacheLookup,httpRequest.cacheHit,httpRequest.cacheValidatedWithOriginServer,httpRequest.cacheFillBytes,httpRequest.protocol,operation.id,operation.producer,operation.first,operation.last,trace,spanId,traceSampled,sourceLocation.file,sourceLocation.line,sourceLocation.function\"}, \"lastUpdatedTimestamp\": 1623969379255}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"1201\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_23abc9e6_17a1c1ee308\", \"text\": \"select * from `harshal-playground-306419.test_schema.austin311_derived`\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"source,city,map_tile,longitude,state_plane_y_coordinate,map_page,complaint_description,status_change_date,last_update_date,latitude,incident_zip,status,created_date,complaint_type,county,owning_department,street_name,unique_key,close_date,street_number,incident_address,state_plane_x_coordinate,council_district_code,location\"}, \"lastUpdatedTimestamp\": 1623969426634}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419._d60e97aec7f471046a960419adb6d44e98300db7.anon26ceee65_7d2c_4522_9d58_0e0966cff37b,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"GET_QUERY_RESULTS_REQUEST\", \"fieldsRead\": \"unique_key,complaint_type,complaint_description,owning_department,source,status,status_change_date,created_date,last_update_date,close_date,incident_address,street_number,street_name,city,incident_zip,county,state_plane_x_coordinate,state_plane_y_coordinate,latitude,longitude,location,council_district_code,map_page,map_tile\"}, \"lastUpdatedTimestamp\": 1623969426871}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"SELECT\", \"numAffectedRows\": 10, \"affectedDatasets\": [\"urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)\"], \"customProperties\": {\"millisecondsTaken\": \"775\", \"sessionId\": \"projects/harshal-playground-306419/jobs/bquxjob_56cdb76b_17a1c1f5123\", \"text\": \"select complaint_description, complaint_type, unique_key, last_update_date from `harshal-playground-306419.test_schema.austin311_derived`\", \"bytesProcessed\": \"10485760\", \"readReason\": \"JOB\", \"fieldsRead\": \"complaint_type,last_update_date,unique_key,complaint_description\"}, \"lastUpdatedTimestamp\": 1623969453887}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419._d60e97aec7f471046a960419adb6d44e98300db7.anon26ceee65_7d2c_4522_9d58_0e0966cff37b,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1626739200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:harshal\", \"operationType\": \"CUSTOM\", \"customOperationType\": \"CUSTOM_READ\", \"affectedDatasets\": [], \"customProperties\": {\"readReason\": \"GET_QUERY_RESULTS_REQUEST\", \"fieldsRead\": \"complaint_description,complaint_type,unique_key,last_update_date\"}, \"lastUpdatedTimestamp\": 1623969454072}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622073600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 0, \"topSqlQueries\": [], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 1, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"unique_key\", \"count\": 1}, {\"fieldPath\": \"complaint_type\", \"count\": 1}, {\"fieldPath\": \"complaint_description\", \"count\": 1}, {\"fieldPath\": \"owning_department\", \"count\": 1}, {\"fieldPath\": \"source\", \"count\": 1}, {\"fieldPath\": \"status\", \"count\": 1}, {\"fieldPath\": \"status_change_date\", \"count\": 1}, {\"fieldPath\": \"created_date\", \"count\": 1}, {\"fieldPath\": \"last_update_date\", \"count\": 1}, {\"fieldPath\": \"close_date\", \"count\": 1}, {\"fieldPath\": \"incident_address\", \"count\": 1}, {\"fieldPath\": \"street_number\", \"count\": 1}, {\"fieldPath\": \"street_name\", \"count\": 1}, {\"fieldPath\": \"city\", \"count\": 1}, {\"fieldPath\": \"incident_zip\", \"count\": 1}, {\"fieldPath\": \"county\", \"count\": 1}, {\"fieldPath\": \"state_plane_x_coordinate\", \"count\": 1}, {\"fieldPath\": \"state_plane_y_coordinate\", \"count\": 1}, {\"fieldPath\": \"latitude\", \"count\": 1}, {\"fieldPath\": \"longitude\", \"count\": 1}, {\"fieldPath\": \"location\", \"count\": 1}, {\"fieldPath\": \"council_district_code\", \"count\": 1}, {\"fieldPath\": \"map_page\", \"count\": 1}, {\"fieldPath\": \"map_tile\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622073600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 3, \"topSqlQueries\": [\"SELECT * FROM `harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_data_access_20210526`\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 9, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"logName\", \"count\": 9}, {\"fieldPath\": \"resource.type\", \"count\": 9}, {\"fieldPath\": \"resource.labels.project_id\", \"count\": 9}, {\"fieldPath\": \"resource.labels.location\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.serviceName\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.methodName\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.resourceName\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.currentLocations\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.originalLocations\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.numResponseItems\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.status.code\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.status.message\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalEmail\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.authoritySelector\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountKeyName\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalSubject\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resource\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.permission\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.granted\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.service\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.name\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.type\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.createTime\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.etag\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.location\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerIp\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerSuppliedUserAgent\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerNetwork\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.id\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.method\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.key\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.value\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.path\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.host\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.scheme\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.query\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.time\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.size\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.protocol\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.reason\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.principal\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.ip\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.port\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.key\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.value\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.principal\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.metadataJson\", \"count\": 9}, {\"fieldPath\": \"textPayload\", \"count\": 9}, {\"fieldPath\": \"timestamp\", \"count\": 9}, {\"fieldPath\": \"receiveTimestamp\", \"count\": 9}, {\"fieldPath\": \"severity\", \"count\": 9}, {\"fieldPath\": \"insertId\", \"count\": 9}, {\"fieldPath\": \"httpRequest.requestMethod\", \"count\": 9}, {\"fieldPath\": \"httpRequest.requestUrl\", \"count\": 9}, {\"fieldPath\": \"httpRequest.requestSize\", \"count\": 9}, {\"fieldPath\": \"httpRequest.status\", \"count\": 9}, {\"fieldPath\": \"httpRequest.responseSize\", \"count\": 9}, {\"fieldPath\": \"httpRequest.userAgent\", \"count\": 9}, {\"fieldPath\": \"httpRequest.remoteIp\", \"count\": 9}, {\"fieldPath\": \"httpRequest.serverIp\", \"count\": 9}, {\"fieldPath\": \"httpRequest.referer\", \"count\": 9}, {\"fieldPath\": \"httpRequest.cacheLookup\", \"count\": 9}, {\"fieldPath\": \"httpRequest.cacheHit\", \"count\": 9}, {\"fieldPath\": \"httpRequest.cacheValidatedWithOriginServer\", \"count\": 9}, {\"fieldPath\": \"httpRequest.cacheFillBytes\", \"count\": 9}, {\"fieldPath\": \"httpRequest.protocol\", \"count\": 9}, {\"fieldPath\": \"operation.id\", \"count\": 9}, {\"fieldPath\": \"operation.producer\", \"count\": 9}, {\"fieldPath\": \"operation.first\", \"count\": 9}, {\"fieldPath\": \"operation.last\", \"count\": 9}, {\"fieldPath\": \"trace\", \"count\": 9}, {\"fieldPath\": \"spanId\", \"count\": 9}, {\"fieldPath\": \"traceSampled\", \"count\": 9}, {\"fieldPath\": \"sourceLocation.file\", \"count\": 9}, {\"fieldPath\": \"sourceLocation.line\", \"count\": 9}, {\"fieldPath\": \"sourceLocation.function\", \"count\": 9}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes\", \"count\": 3}, {\"fieldPath\": \"operation\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog\", \"count\": 3}, {\"fieldPath\": \"resource.labels\", \"count\": 3}, {\"fieldPath\": \"sourceLocation\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.status\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation\", \"count\": 3}, {\"fieldPath\": \"resource\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata\", \"count\": 3}, {\"fieldPath\": \"httpRequest\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels\", \"count\": 3}, {\"fieldPath\": \"resource.labels.dataset_id\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId\", \"count\": 2}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields\", \"count\": 2}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622073600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 1, \"topSqlQueries\": [\"SELECT * FROM `harshal-playground-306419.bigquery_usage_logs.cloudaudit_googleapis_com_activity_20210527` LIMIT 1000\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 10, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"logName\", \"count\": 10}, {\"fieldPath\": \"resource.type\", \"count\": 10}, {\"fieldPath\": \"resource.labels.project_id\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.serviceName\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.methodName\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.resourceName\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.currentLocations\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.originalLocations\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.numResponseItems\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.status.code\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.status.message\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalEmail\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.authoritySelector\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountKeyName\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalSubject\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resource\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.permission\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.granted\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.service\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.name\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.type\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.createTime\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.etag\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.location\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerIp\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerSuppliedUserAgent\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerNetwork\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.id\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.method\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.key\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.value\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.path\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.host\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.scheme\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.query\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.time\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.size\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.protocol\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.reason\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.principal\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.ip\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.port\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.key\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.value\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.principal\", \"count\": 10}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode\", \"count\": 10}, {\"fieldPath\": \"textPayload\", \"count\": 10}, {\"fieldPath\": \"timestamp\", \"count\": 10}, {\"fieldPath\": \"receiveTimestamp\", \"count\": 10}, {\"fieldPath\": \"severity\", \"count\": 10}, {\"fieldPath\": \"insertId\", \"count\": 10}, {\"fieldPath\": \"httpRequest.requestMethod\", \"count\": 10}, {\"fieldPath\": \"httpRequest.requestUrl\", \"count\": 10}, {\"fieldPath\": \"httpRequest.requestSize\", \"count\": 10}, {\"fieldPath\": \"httpRequest.status\", \"count\": 10}, {\"fieldPath\": \"httpRequest.responseSize\", \"count\": 10}, {\"fieldPath\": \"httpRequest.userAgent\", \"count\": 10}, {\"fieldPath\": \"httpRequest.remoteIp\", \"count\": 10}, {\"fieldPath\": \"httpRequest.serverIp\", \"count\": 10}, {\"fieldPath\": \"httpRequest.referer\", \"count\": 10}, {\"fieldPath\": \"httpRequest.cacheLookup\", \"count\": 10}, {\"fieldPath\": \"httpRequest.cacheHit\", \"count\": 10}, {\"fieldPath\": \"httpRequest.cacheValidatedWithOriginServer\", \"count\": 10}, {\"fieldPath\": \"httpRequest.cacheFillBytes\", \"count\": 10}, {\"fieldPath\": \"httpRequest.protocol\", \"count\": 10}, {\"fieldPath\": \"operation.id\", \"count\": 10}, {\"fieldPath\": \"operation.producer\", \"count\": 10}, {\"fieldPath\": \"operation.first\", \"count\": 10}, {\"fieldPath\": \"operation.last\", \"count\": 10}, {\"fieldPath\": \"trace\", \"count\": 10}, {\"fieldPath\": \"spanId\", \"count\": 10}, {\"fieldPath\": \"traceSampled\", \"count\": 10}, {\"fieldPath\": \"sourceLocation.file\", \"count\": 10}, {\"fieldPath\": \"sourceLocation.line\", \"count\": 10}, {\"fieldPath\": \"sourceLocation.function\", \"count\": 10}, {\"fieldPath\": \"resource.labels.dataset_id\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.metadataJson\", \"count\": 7}, {\"fieldPath\": \"resource.labels.name\", \"count\": 3}, {\"fieldPath\": \"resource.labels.destination\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestJson\", \"count\": 3}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal\", \"count\": 1}, {\"fieldPath\": \"resource.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo\", \"count\": 1}, {\"fieldPath\": \"operation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels\", \"count\": 1}, {\"fieldPath\": \"sourceLocation\", \"count\": 1}, {\"fieldPath\": \"httpRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog\", \"count\": 1}, {\"fieldPath\": \"resource\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622073600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 4, \"topSqlQueries\": [\"\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\\n\", \"SELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 4, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"placename\", \"count\": 4}, {\"fieldPath\": \"excess_deaths\", \"count\": 4}, {\"fieldPath\": \"deaths\", \"count\": 4}, {\"fieldPath\": \"end_date\", \"count\": 4}, {\"fieldPath\": \"frequency\", \"count\": 4}, {\"fieldPath\": \"expected_deaths\", \"count\": 4}, {\"fieldPath\": \"start_date\", \"count\": 4}, {\"fieldPath\": \"baseline\", \"count\": 4}, {\"fieldPath\": \"year\", \"count\": 4}, {\"fieldPath\": \"month\", \"count\": 4}, {\"fieldPath\": \"week\", \"count\": 4}, {\"fieldPath\": \"country\", \"count\": 4}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622073600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 1, \"topSqlQueries\": [\"SELECT * FROM `harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity` WHERE DATE(timestamp) = \\\"2021-05-27\\\" LIMIT 1000\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 4, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"logName\", \"count\": 4}, {\"fieldPath\": \"resource.type\", \"count\": 4}, {\"fieldPath\": \"resource.labels.project_id\", \"count\": 4}, {\"fieldPath\": \"resource.labels.dataset_id\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.serviceName\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.methodName\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.resourceName\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.currentLocations\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.originalLocations\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.numResponseItems\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.status.code\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.status.message\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalEmail\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.authoritySelector\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountKeyName\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalSubject\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resource\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.permission\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.granted\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.service\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.name\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.type\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.createTime\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.etag\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.location\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerIp\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerSuppliedUserAgent\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerNetwork\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.id\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.method\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.key\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.value\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.path\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.host\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.scheme\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.query\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.time\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.size\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.protocol\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.reason\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.principal\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.ip\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.port\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.key\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.value\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.principal\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.metadataJson\", \"count\": 4}, {\"fieldPath\": \"textPayload\", \"count\": 4}, {\"fieldPath\": \"timestamp\", \"count\": 4}, {\"fieldPath\": \"receiveTimestamp\", \"count\": 4}, {\"fieldPath\": \"severity\", \"count\": 4}, {\"fieldPath\": \"insertId\", \"count\": 4}, {\"fieldPath\": \"httpRequest.requestMethod\", \"count\": 4}, {\"fieldPath\": \"httpRequest.requestUrl\", \"count\": 4}, {\"fieldPath\": \"httpRequest.requestSize\", \"count\": 4}, {\"fieldPath\": \"httpRequest.status\", \"count\": 4}, {\"fieldPath\": \"httpRequest.responseSize\", \"count\": 4}, {\"fieldPath\": \"httpRequest.userAgent\", \"count\": 4}, {\"fieldPath\": \"httpRequest.remoteIp\", \"count\": 4}, {\"fieldPath\": \"httpRequest.serverIp\", \"count\": 4}, {\"fieldPath\": \"httpRequest.referer\", \"count\": 4}, {\"fieldPath\": \"httpRequest.cacheLookup\", \"count\": 4}, {\"fieldPath\": \"httpRequest.cacheHit\", \"count\": 4}, {\"fieldPath\": \"httpRequest.cacheValidatedWithOriginServer\", \"count\": 4}, {\"fieldPath\": \"httpRequest.cacheFillBytes\", \"count\": 4}, {\"fieldPath\": \"httpRequest.protocol\", \"count\": 4}, {\"fieldPath\": \"operation.id\", \"count\": 4}, {\"fieldPath\": \"operation.producer\", \"count\": 4}, {\"fieldPath\": \"operation.first\", \"count\": 4}, {\"fieldPath\": \"operation.last\", \"count\": 4}, {\"fieldPath\": \"trace\", \"count\": 4}, {\"fieldPath\": \"spanId\", \"count\": 4}, {\"fieldPath\": \"traceSampled\", \"count\": 4}, {\"fieldPath\": \"sourceLocation.file\", \"count\": 4}, {\"fieldPath\": \"sourceLocation.line\", \"count\": 4}, {\"fieldPath\": \"sourceLocation.function\", \"count\": 4}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes\", \"count\": 1}, {\"fieldPath\": \"sourceLocation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo\", \"count\": 1}, {\"fieldPath\": \"operation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels\", \"count\": 1}, {\"fieldPath\": \"resource.labels\", \"count\": 1}, {\"fieldPath\": \"httpRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog\", \"count\": 1}, {\"fieldPath\": \"resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622073600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 1, \"topSqlQueries\": [\"SELECT * FROM `harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_data_access` WHERE DATE(timestamp) = \\\"2021-05-27\\\" LIMIT 1000\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 7, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"logName\", \"count\": 7}, {\"fieldPath\": \"resource.type\", \"count\": 7}, {\"fieldPath\": \"resource.labels.project_id\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.serviceName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.methodName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.resourceName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.currentLocations\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.originalLocations\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.numResponseItems\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.status.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.status.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.authoritySelector\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalSubject\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resource\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.permission\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.granted\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.service\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.type\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.etag\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerIp\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerSuppliedUserAgent\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerNetwork\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.id\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.method\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.path\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.host\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.scheme\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.time\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.size\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.protocol\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.reason\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.principal\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.ip\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.port\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.principal\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.expireTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.truncateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.expireTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.truncateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest.listAll\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.role\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.groupEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.userEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.domain\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.specialGroup\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.role\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.groupEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.userEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.domain\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.specialGroup\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.jobId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.queryPriority\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.statementType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.destinationUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.state\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.startTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.endTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalProcessedBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalBilledBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.billingTier\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalSlotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage.slotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalTablesProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalViewsProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.queryOutputRowCount\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.totalLoadOutputBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.maxResults\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.maxResults\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest.startRow\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.startRow\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest.maxResults\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.resource\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.version\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.role\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.members\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.expression\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.title\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.service\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.logType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs.exemptedMembers\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.etag\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask.paths\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.expireTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.truncateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.expireTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.truncateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.role\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.groupEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.userEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.domain\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.specialGroup\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.friendlyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.updateTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.role\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.groupEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.userEmail\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.domain\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.specialGroup\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.jobId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.queryPriority\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.statementType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.destinationUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.state\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.startTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.endTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalProcessedBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalBilledBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.billingTier\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalSlotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage.slotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalTablesProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalViewsProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.queryOutputRowCount\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.totalLoadOutputBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.totalResults\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.jobId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.queryPriority\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.statementType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.destinationUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.state\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.startTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.endTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalProcessedBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalBilledBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.billingTier\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalSlotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage.slotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalTablesProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalViewsProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.queryOutputRowCount\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.totalLoadOutputBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.totalResults\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.jobId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.queryPriority\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.statementType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.destinationUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.state\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.startTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.endTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalProcessedBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalBilledBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.billingTier\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalSlotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage.slotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalTablesProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalViewsProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.queryOutputRowCount\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.totalLoadOutputBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.jobId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.queryPriority\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.statementType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.destinationUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.state\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.startTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.endTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalProcessedBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalBilledBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.billingTier\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalSlotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage.slotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalTablesProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalViewsProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.queryOutputRowCount\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.totalLoadOutputBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.version\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.role\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.members\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.expression\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.title\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.description\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.service\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.logType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs.exemptedMembers\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.etag\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.eventName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.location\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.queryPriority\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.statementType\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.sourceUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.schemaJson\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.destinationUris\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.createDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.writeDisposition\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption.kmsKeyName\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.dryRun\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.key\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels.value\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.code\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors.message\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.createTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.startTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.endTime\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalProcessedBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.billingTier\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalSlotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.name\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage.slotMs\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalTablesProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalViewsProcessed\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.queryOutputRowCount\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalLoadOutputBytes\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.projectId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.datasetId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName.tableId\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.referencedFields\", \"count\": 7}, {\"fieldPath\": \"textPayload\", \"count\": 7}, {\"fieldPath\": \"timestamp\", \"count\": 7}, {\"fieldPath\": \"receiveTimestamp\", \"count\": 7}, {\"fieldPath\": \"severity\", \"count\": 7}, {\"fieldPath\": \"insertId\", \"count\": 7}, {\"fieldPath\": \"httpRequest.requestMethod\", \"count\": 7}, {\"fieldPath\": \"httpRequest.requestUrl\", \"count\": 7}, {\"fieldPath\": \"httpRequest.requestSize\", \"count\": 7}, {\"fieldPath\": \"httpRequest.status\", \"count\": 7}, {\"fieldPath\": \"httpRequest.responseSize\", \"count\": 7}, {\"fieldPath\": \"httpRequest.userAgent\", \"count\": 7}, {\"fieldPath\": \"httpRequest.remoteIp\", \"count\": 7}, {\"fieldPath\": \"httpRequest.serverIp\", \"count\": 7}, {\"fieldPath\": \"httpRequest.referer\", \"count\": 7}, {\"fieldPath\": \"httpRequest.cacheLookup\", \"count\": 7}, {\"fieldPath\": \"httpRequest.cacheHit\", \"count\": 7}, {\"fieldPath\": \"httpRequest.cacheValidatedWithOriginServer\", \"count\": 7}, {\"fieldPath\": \"httpRequest.cacheFillBytes\", \"count\": 7}, {\"fieldPath\": \"httpRequest.protocol\", \"count\": 7}, {\"fieldPath\": \"operation.id\", \"count\": 7}, {\"fieldPath\": \"operation.producer\", \"count\": 7}, {\"fieldPath\": \"operation.first\", \"count\": 7}, {\"fieldPath\": \"operation.last\", \"count\": 7}, {\"fieldPath\": \"trace\", \"count\": 7}, {\"fieldPath\": \"spanId\", \"count\": 7}, {\"fieldPath\": \"traceSampled\", \"count\": 7}, {\"fieldPath\": \"sourceLocation.file\", \"count\": 7}, {\"fieldPath\": \"sourceLocation.line\", \"count\": 7}, {\"fieldPath\": \"sourceLocation.function\", \"count\": 7}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries.viewName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.reservationUsage\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract.sourceTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.reservationUsage\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedViews\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.additionalErrors\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.tableName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.error\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedViews\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.tableDefinitions\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.referencedTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.tableDefinitions\", \"count\": 1}, {\"fieldPath\": \"resource.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings.condition\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.sourceTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.load.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"sourceLocation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.extract.sourceTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.view\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.extract\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract.sourceTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest\", \"count\": 1}, {\"fieldPath\": \"httpRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataListRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.view\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.sourceTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedViews\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries.viewName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.reservationUsage\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetListRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs.auditLogConfigs\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableDataReadEvents.tableName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.extract\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.datasetName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.datasetName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.additionalErrors\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.tableCopy.sourceTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.encryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.datasetName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.error\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl.entries.viewName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.tableDefinitions\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.encryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics.reservationUsage\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource.encryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics.referencedViews\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.sourceTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.referencedTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.reservationUsage\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.auditConfigs.auditLogConfigs\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.defaultDataset\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertResponse.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.encryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatus.error\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatistics.reservationUsage\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.bindings\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.query.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.referencedTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertRequest.resource.acl.entries.viewName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.datasetName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.sourceTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.load.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatus.additionalErrors\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.tableName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobStatistics.referencedViews\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedViews\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract.sourceTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract.sourceTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.tableName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.load\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.policy.auditConfigs\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobStatistics\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource.view\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.load.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.tableDefinitions\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.view\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateRequest.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.error\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.error\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatistics.referencedTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.acl.entries\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus.additionalErrors\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobStatus\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobGetQueryResultsResponse.job.jobConfiguration.query.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.setIamPolicyRequest.updateMask\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.tableCopy.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableUpdateResponse.resource.tableName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings.condition\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateResponse.resource.info\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus\", \"count\": 1}, {\"fieldPath\": \"operation\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.extract\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.tableCopy.sourceTables\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatistics\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.query.tableDefinitions\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.load\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobStatus.additionalErrors\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.info.labels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryDoneResponse.job.jobConfiguration.query.tableDefinitions\", \"count\": 1}, {\"fieldPath\": \"resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.tableCopy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.tableCopy\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.policyResponse.bindings\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobStatus.additionalErrors\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertRequest.resource.jobConfiguration.extract\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetInsertResponse.resource.acl\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.tableCopy.destinationTableEncryption\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.extract.sourceTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobInsertResponse.resource.jobConfiguration.load\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.tableInsertRequest.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.jobQueryResponse.job.jobConfiguration.query.destinationTable\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.servicedata_v1_bigquery.datasetUpdateRequest.resource.acl.entries\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622160000000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 0, \"topSqlQueries\": [], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 1, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"logName\", \"count\": 1}, {\"fieldPath\": \"resource.type\", \"count\": 1}, {\"fieldPath\": \"resource.labels.project_id\", \"count\": 1}, {\"fieldPath\": \"resource.labels.dataset_id\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.serviceName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.methodName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.currentLocations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.originalLocations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.numResponseItems\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status.code\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status.message\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalEmail\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.authoritySelector\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountKeyName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalSubject\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.permission\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.granted\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.service\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.name\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.type\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.createTime\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.etag\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.location\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerIp\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerSuppliedUserAgent\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerNetwork\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.id\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.method\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.path\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.host\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.scheme\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.time\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.size\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.protocol\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.reason\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.principal\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.ip\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.port\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.principal\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.metadataJson\", \"count\": 1}, {\"fieldPath\": \"textPayload\", \"count\": 1}, {\"fieldPath\": \"timestamp\", \"count\": 1}, {\"fieldPath\": \"receiveTimestamp\", \"count\": 1}, {\"fieldPath\": \"severity\", \"count\": 1}, {\"fieldPath\": \"insertId\", \"count\": 1}, {\"fieldPath\": \"httpRequest.requestMethod\", \"count\": 1}, {\"fieldPath\": \"httpRequest.requestUrl\", \"count\": 1}, {\"fieldPath\": \"httpRequest.requestSize\", \"count\": 1}, {\"fieldPath\": \"httpRequest.status\", \"count\": 1}, {\"fieldPath\": \"httpRequest.responseSize\", \"count\": 1}, {\"fieldPath\": \"httpRequest.userAgent\", \"count\": 1}, {\"fieldPath\": \"httpRequest.remoteIp\", \"count\": 1}, {\"fieldPath\": \"httpRequest.serverIp\", \"count\": 1}, {\"fieldPath\": \"httpRequest.referer\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheLookup\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheHit\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheValidatedWithOriginServer\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheFillBytes\", \"count\": 1}, {\"fieldPath\": \"httpRequest.protocol\", \"count\": 1}, {\"fieldPath\": \"operation.id\", \"count\": 1}, {\"fieldPath\": \"operation.producer\", \"count\": 1}, {\"fieldPath\": \"operation.first\", \"count\": 1}, {\"fieldPath\": \"operation.last\", \"count\": 1}, {\"fieldPath\": \"trace\", \"count\": 1}, {\"fieldPath\": \"spanId\", \"count\": 1}, {\"fieldPath\": \"traceSampled\", \"count\": 1}, {\"fieldPath\": \"sourceLocation.file\", \"count\": 1}, {\"fieldPath\": \"sourceLocation.line\", \"count\": 1}, {\"fieldPath\": \"sourceLocation.function\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622160000000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 2, \"topSqlQueries\": [\"SELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`\", \"# CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 2, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"deaths\", \"count\": 2}, {\"fieldPath\": \"end_date\", \"count\": 2}, {\"fieldPath\": \"excess_deaths\", \"count\": 2}, {\"fieldPath\": \"frequency\", \"count\": 2}, {\"fieldPath\": \"placename\", \"count\": 2}, {\"fieldPath\": \"start_date\", \"count\": 2}, {\"fieldPath\": \"week\", \"count\": 2}, {\"fieldPath\": \"baseline\", \"count\": 2}, {\"fieldPath\": \"country\", \"count\": 2}, {\"fieldPath\": \"month\", \"count\": 2}, {\"fieldPath\": \"expected_deaths\", \"count\": 2}, {\"fieldPath\": \"year\", \"count\": 2}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622505600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 1, \"topSqlQueries\": [\"# CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.excess_deaths_derived`;\\n\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 1, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"excess_deaths\", \"count\": 1}, {\"fieldPath\": \"frequency\", \"count\": 1}, {\"fieldPath\": \"deaths\", \"count\": 1}, {\"fieldPath\": \"baseline\", \"count\": 1}, {\"fieldPath\": \"week\", \"count\": 1}, {\"fieldPath\": \"month\", \"count\": 1}, {\"fieldPath\": \"placename\", \"count\": 1}, {\"fieldPath\": \"country\", \"count\": 1}, {\"fieldPath\": \"start_date\", \"count\": 1}, {\"fieldPath\": \"year\", \"count\": 1}, {\"fieldPath\": \"expected_deaths\", \"count\": 1}, {\"fieldPath\": \"end_date\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1622505600000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 1, \"topSqlQueries\": [\"# CREATE OR REPLACE TABLE test_schema.excess_deaths_derived AS (SELECT * FROM `bigquery-public-data.covid19_nyt.excess_deaths` LIMIT 10);\\n\\nSELECT * FROM `harshal-playground-306419.test_schema.austin311_derived`;\\n\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 1, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"last_update_date\", \"count\": 1}, {\"fieldPath\": \"created_date\", \"count\": 1}, {\"fieldPath\": \"location\", \"count\": 1}, {\"fieldPath\": \"city\", \"count\": 1}, {\"fieldPath\": \"complaint_type\", \"count\": 1}, {\"fieldPath\": \"status_change_date\", \"count\": 1}, {\"fieldPath\": \"street_number\", \"count\": 1}, {\"fieldPath\": \"state_plane_x_coordinate\", \"count\": 1}, {\"fieldPath\": \"incident_zip\", \"count\": 1}, {\"fieldPath\": \"source\", \"count\": 1}, {\"fieldPath\": \"council_district_code\", \"count\": 1}, {\"fieldPath\": \"map_tile\", \"count\": 1}, {\"fieldPath\": \"state_plane_y_coordinate\", \"count\": 1}, {\"fieldPath\": \"map_page\", \"count\": 1}, {\"fieldPath\": \"unique_key\", \"count\": 1}, {\"fieldPath\": \"owning_department\", \"count\": 1}, {\"fieldPath\": \"street_name\", \"count\": 1}, {\"fieldPath\": \"complaint_description\", \"count\": 1}, {\"fieldPath\": \"longitude\", \"count\": 1}, {\"fieldPath\": \"incident_address\", \"count\": 1}, {\"fieldPath\": \"status\", \"count\": 1}, {\"fieldPath\": \"close_date\", \"count\": 1}, {\"fieldPath\": \"latitude\", \"count\": 1}, {\"fieldPath\": \"county\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.bq_audit.cloudaudit_googleapis_com_activity,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1623888000000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 0, \"topSqlQueries\": [], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 1, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"logName\", \"count\": 1}, {\"fieldPath\": \"resource.type\", \"count\": 1}, {\"fieldPath\": \"resource.labels.project_id\", \"count\": 1}, {\"fieldPath\": \"resource.labels.dataset_id\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.serviceName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.methodName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.currentLocations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.resourceLocation.originalLocations\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.numResponseItems\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status.code\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.status.message\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalEmail\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.authoritySelector\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountKeyName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.principalSubject\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.serviceAccountDelegationInfo.firstPartyPrincipal.principalEmail\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authenticationInfo.principalSubject\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resource\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.permission\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.granted\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.service\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.name\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.type\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.labels.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.uid\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.annotations.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.displayName\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.createTime\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.updateTime\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.deleteTime\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.etag\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.authorizationInfo.resourceAttributes.location\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerIp\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerSuppliedUserAgent\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.callerNetwork\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.id\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.method\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.headers.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.path\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.host\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.scheme\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.query\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.time\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.size\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.protocol\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.reason\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.principal\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.audiences\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.presenter\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.requestAttributes.auth.accessLevels\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.ip\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.port\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.key\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.labels.value\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.principal\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.requestMetadata.destinationAttributes.regionCode\", \"count\": 1}, {\"fieldPath\": \"protopayload_auditlog.metadataJson\", \"count\": 1}, {\"fieldPath\": \"textPayload\", \"count\": 1}, {\"fieldPath\": \"timestamp\", \"count\": 1}, {\"fieldPath\": \"receiveTimestamp\", \"count\": 1}, {\"fieldPath\": \"severity\", \"count\": 1}, {\"fieldPath\": \"insertId\", \"count\": 1}, {\"fieldPath\": \"httpRequest.requestMethod\", \"count\": 1}, {\"fieldPath\": \"httpRequest.requestUrl\", \"count\": 1}, {\"fieldPath\": \"httpRequest.requestSize\", \"count\": 1}, {\"fieldPath\": \"httpRequest.status\", \"count\": 1}, {\"fieldPath\": \"httpRequest.responseSize\", \"count\": 1}, {\"fieldPath\": \"httpRequest.userAgent\", \"count\": 1}, {\"fieldPath\": \"httpRequest.remoteIp\", \"count\": 1}, {\"fieldPath\": \"httpRequest.serverIp\", \"count\": 1}, {\"fieldPath\": \"httpRequest.referer\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheLookup\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheHit\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheValidatedWithOriginServer\", \"count\": 1}, {\"fieldPath\": \"httpRequest.cacheFillBytes\", \"count\": 1}, {\"fieldPath\": \"httpRequest.protocol\", \"count\": 1}, {\"fieldPath\": \"operation.id\", \"count\": 1}, {\"fieldPath\": \"operation.producer\", \"count\": 1}, {\"fieldPath\": \"operation.first\", \"count\": 1}, {\"fieldPath\": \"operation.last\", \"count\": 1}, {\"fieldPath\": \"trace\", \"count\": 1}, {\"fieldPath\": \"spanId\", \"count\": 1}, {\"fieldPath\": \"traceSampled\", \"count\": 1}, {\"fieldPath\": \"sourceLocation.file\", \"count\": 1}, {\"fieldPath\": \"sourceLocation.line\", \"count\": 1}, {\"fieldPath\": \"sourceLocation.function\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.austin311_derived,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1623888000000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 2, \"topSqlQueries\": [\"select * from `harshal-playground-306419.test_schema.austin311_derived`\", \"select complaint_description, complaint_type, unique_key, last_update_date from `harshal-playground-306419.test_schema.austin311_derived`\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:harshal\", \"count\": 2, \"userEmail\": \"harshal@acryl.io\"}], \"fieldCounts\": [{\"fieldPath\": \"complaint_description\", \"count\": 2}, {\"fieldPath\": \"last_update_date\", \"count\": 2}, {\"fieldPath\": \"complaint_type\", \"count\": 2}, {\"fieldPath\": \"unique_key\", \"count\": 2}, {\"fieldPath\": \"source\", \"count\": 1}, {\"fieldPath\": \"city\", \"count\": 1}, {\"fieldPath\": \"map_tile\", \"count\": 1}, {\"fieldPath\": \"longitude\", \"count\": 1}, {\"fieldPath\": \"state_plane_y_coordinate\", \"count\": 1}, {\"fieldPath\": \"map_page\", \"count\": 1}, {\"fieldPath\": \"status_change_date\", \"count\": 1}, {\"fieldPath\": \"latitude\", \"count\": 1}, {\"fieldPath\": \"incident_zip\", \"count\": 1}, {\"fieldPath\": \"status\", \"count\": 1}, {\"fieldPath\": \"created_date\", \"count\": 1}, {\"fieldPath\": \"county\", \"count\": 1}, {\"fieldPath\": \"owning_department\", \"count\": 1}, {\"fieldPath\": \"street_name\", \"count\": 1}, {\"fieldPath\": \"close_date\", \"count\": 1}, {\"fieldPath\": \"street_number\", \"count\": 1}, {\"fieldPath\": \"incident_address\", \"count\": 1}, {\"fieldPath\": \"state_plane_x_coordinate\", \"count\": 1}, {\"fieldPath\": \"council_district_code\", \"count\": 1}, {\"fieldPath\": \"location\", \"count\": 1}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1626739200000, + "runId": "test-bigquery-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/bigquery-usage/test_bigquery_usage.py b/metadata-ingestion/tests/integration/bigquery-usage/test_bigquery_usage.py index db80fc641c7bd4..4282545a2f802c 100644 --- a/metadata-ingestion/tests/integration/bigquery-usage/test_bigquery_usage.py +++ b/metadata-ingestion/tests/integration/bigquery-usage/test_bigquery_usage.py @@ -117,14 +117,91 @@ def test_bq_usage_source(pytestconfig, tmp_path): ) +@freeze_time(FROZEN_TIME) +def test_bq_usage_source_with_read_events(pytestconfig, tmp_path): + # from google.cloud.logging_v2 import ProtobufEntry + + test_resources_dir: pathlib.Path = ( + pytestconfig.rootpath / "tests/integration/bigquery-usage" + ) + bigquery_reference_logs_path = test_resources_dir / "bigquery_logs.json" + + if WRITE_REFERENCE_FILE: + source = BigQueryUsageSource.create( + dict( + projects=[ + "harshal-playground-306419", + ], + start_time=datetime.now(tz=timezone.utc) - timedelta(days=25), + ), + PipelineContext(run_id="bq-usage-test"), + ) + entries = list( + source._get_bigquery_log_entries_via_gcp_logging( + source._make_bigquery_logging_clients() + ) + ) + + entries = [entry._replace(logger=None) for entry in entries] + log_entries = jsonpickle.encode(entries, indent=4) + with bigquery_reference_logs_path.open("w") as logs: + logs.write(log_entries) + + with unittest.mock.patch( + "datahub.ingestion.source.usage.bigquery_usage.GCPLoggingClient", autospec=True + ) as MockClient: + # Add mock BigQuery API responses. + with bigquery_reference_logs_path.open() as logs: + reference_logs = jsonpickle.decode(logs.read()) + MockClient().list_entries.return_value = reference_logs + + # Run a BigQuery usage ingestion run. + pipeline = Pipeline.create( + { + "run_id": "test-bigquery-usage", + "source": { + "type": "bigquery-usage", + "config": { + "projects": ["sample-bigquery-project-1234"], + "start_time": "2021-01-01T00:00Z", + "end_time": "2021-07-01T00:00Z", + "include_read_operational_stats": True, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/bigquery_usages_with_read_events.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "bigquery_usages_with_read_events.json", + golden_path=test_resources_dir / "bigquery_usages_with_read_events_golden.json", + ) + + @pytest.mark.parametrize( "test_input,expected", [ ("test_table$20220101", "test_table"), ("test_table$__PARTITIONS_SUMMARY__", "test_table"), ("test_table_20220101", "test_table"), + ("20220101", "test_dataset"), + ("test_table", "test_table"), ], ) def test_remove_extras(test_input, expected): + config = BigQueryUsageConfig.parse_obj( + dict( + project_id="sample-bigquery-project-name-1234", + ) + ) + table_ref = BigQueryTableRef("test_project", "test_dataset", test_input) - assert table_ref.remove_extras().table == expected + assert table_ref.remove_extras(config.sharded_table_pattern).table == expected diff --git a/metadata-ingestion/tests/integration/circuit_breaker/__init__.py b/metadata-ingestion/tests/integration/circuit_breaker/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_empty_response.json b/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_empty_response.json new file mode 100644 index 00000000000000..5f7adc514ef075 --- /dev/null +++ b/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_empty_response.json @@ -0,0 +1,9 @@ +{ + "dataset": { + "assertions": { + "__typename": "EntityAssertionsResult", + "total": 0, + "assertions": [] + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_response.json b/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_response.json new file mode 100644 index 00000000000000..73dc104f59e609 --- /dev/null +++ b/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_response.json @@ -0,0 +1,244 @@ +{ + "dataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "operations": [], + "incidents": { + "incidents": [] + }, + "assertions": { + "total": 5, + "assertions": [ + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 2, + "succeeded": 0, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "FAILURE", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "FAILURE", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": 3, + "missingCount": null, + "unexpectedCount": 0, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": 2, + "missingCount": null, + "unexpectedCount": 0, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 1, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 3, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 3, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 2, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74" + } + ] + } + } + ] + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_response_with_no_error.json b/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_response_with_no_error.json new file mode 100644 index 00000000000000..1d7099c9321605 --- /dev/null +++ b/metadata-ingestion/tests/integration/circuit_breaker/assertion_gql_response_with_no_error.json @@ -0,0 +1,198 @@ +{ + "dataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)", + "operations": [], + "incidents": { + "incidents": [] + }, + "assertions": { + "total": 4, + "assertions": [ + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:3d1699164901675df774ab34fd16f4f3" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": 3, + "missingCount": null, + "unexpectedCount": 0, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": 2, + "missingCount": null, + "unexpectedCount": 0, + "actualAggValue": null, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:9729dfafea4bb2c2f114bc80e513a7ec" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 1, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 3, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:16d6f586b2febda7f2b53faec6bb9035" + } + ] + } + }, + { + "__typename": "Assertion", + "runEvents": { + "total": 2, + "failed": 0, + "succeeded": 2, + "runEvents": [ + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catA\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 3, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74" + }, + { + "__typename": "AssertionRunEvent", + "timestampMillis": 1640692800000, + "partitionSpec": { + "type": "PARTITION", + "partition": "{\"category\": \"catB\"}", + "timePartition": null + }, + "result": { + "type": "SUCCESS", + "rowCount": null, + "missingCount": null, + "unexpectedCount": null, + "actualAggValue": 2, + "externalUrl": null + }, + "assertionUrn": "urn:li:assertion:4cf76385ccf614cc6cbb9daa551c3c74" + } + ] + } + } + ] + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/circuit_breaker/operation_gql_empty_response.json b/metadata-ingestion/tests/integration/circuit_breaker/operation_gql_empty_response.json new file mode 100644 index 00000000000000..0f99316f9ec933 --- /dev/null +++ b/metadata-ingestion/tests/integration/circuit_breaker/operation_gql_empty_response.json @@ -0,0 +1,6 @@ +{ + "dataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "operations": [] + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/circuit_breaker/operation_gql_response.json b/metadata-ingestion/tests/integration/circuit_breaker/operation_gql_response.json new file mode 100644 index 00000000000000..6f52f95b115a36 --- /dev/null +++ b/metadata-ingestion/tests/integration/circuit_breaker/operation_gql_response.json @@ -0,0 +1,79 @@ +{ + "dataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.jaffle_shop.customers,PROD)", + "operations": [ + { + "__typename": "Operation", + "actor": "urn:li:corpuser:bq-usage", + "operationType": "CUSTOM", + "sourceType": null, + "numAffectedRows": 1, + "partition": "FULL_TABLE_SNAPSHOT", + "timestampMillis": 1655769674092, + "lastUpdatedTimestamp": 1655700504432, + "customProperties": [ + { + "key": "millisecondsTaken", + "value": "222" + }, + { + "key": "text", + "value": "/* {\"app\": \"dbt\", \"dbt_version\": \"1.1.0\", \"profile_name\": \"jaffle_shop\", \"target_name\": \"dev\", \"node_id\": \"test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1\"} */\nselect\n count(*) as failures,\n count(*) != 0 as should_warn,\n count(*) != 0 as should_error\n from (\n \n \n \n\nwith dbt_test__target as (\n\n select customer_id as unique_field\n from `my_project`.`jaffle_shop`.`customers`\n where customer_id is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n\n\n \n ) dbt_internal_test" + }, + { + "key": "sessionId", + "value": "projects/my_project/jobs/b68487dc-61db-4f01-abd7-c5f7d931a46c" + }, + { + "key": "fieldsRead", + "value": "customer_id" + }, + { + "key": "readReason", + "value": "JOB" + }, + { + "key": "bytesProcessed", + "value": "10485760" + } + ] + }, + { + "__typename": "Operation", + "actor": "urn:li:corpuser:bq-usage", + "operationType": "CUSTOM", + "sourceType": null, + "numAffectedRows": 1, + "partition": "FULL_TABLE_SNAPSHOT", + "timestampMillis": 1655769674090, + "lastUpdatedTimestamp": 1655700503898, + "customProperties": [ + { + "key": "millisecondsTaken", + "value": "234" + }, + { + "key": "text", + "value": "/* {\"app\": \"dbt\", \"dbt_version\": \"1.1.0\", \"profile_name\": \"jaffle_shop\", \"target_name\": \"dev\", \"node_id\": \"test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2\"} */\nselect\n count(*) as failures,\n count(*) != 0 as should_warn,\n count(*) != 0 as should_error\n from (\n \n \n \n\nwith child as (\n select customer_id as from_field\n from `my_project`.`jaffle_shop`.`orders`\n where customer_id is not null\n),\n\nparent as (\n select customer_id as to_field\n from `my_project`.`jaffle_shop`.`customers`\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n\n\n \n ) dbt_internal_test" + }, + { + "key": "sessionId", + "value": "projects/my_project/jobs/4b6ae0b9-b7d3-43d4-aaae-1baf91be3553" + }, + { + "key": "fieldsRead", + "value": "customer_id" + }, + { + "key": "readReason", + "value": "JOB" + }, + { + "key": "bytesProcessed", + "value": "20971520" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/circuit_breaker/test_circuit_breaker.py b/metadata-ingestion/tests/integration/circuit_breaker/test_circuit_breaker.py new file mode 100644 index 00000000000000..b9c661935c5e06 --- /dev/null +++ b/metadata-ingestion/tests/integration/circuit_breaker/test_circuit_breaker.py @@ -0,0 +1,155 @@ +import json +from unittest.mock import patch + +import pytest +from freezegun import freeze_time + +try: + from datahub.api.circuit_breaker import ( + AssertionCircuitBreaker, + AssertionCircuitBreakerConfig, + OperationCircuitBreaker, + OperationCircuitBreakerConfig, + ) +# Imports are only available if we are running integrations tests +except ImportError: + pass +lastUpdatedResponseBeforeLastAssertion = { + "dataset": {"operations": [{"lastUpdatedTimestamp": 1640685600000}]} +} + +lastUpdatedResponseAfterLastAssertion = { + "dataset": {"operations": [{"lastUpdatedTimestamp": 1652450039000}]} +} + + +@pytest.mark.integration +def test_operation_circuit_breaker_with_empty_response(pytestconfig): + with patch("gql.client.Client.execute") as mock_gql_client: + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + f = open( + f"{test_resources_dir}/operation_gql_empty_response.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [data] + + config = OperationCircuitBreakerConfig(datahub_host="dummy") + cb = OperationCircuitBreaker(config) + + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD))" + ) + assert result is True + + +@freeze_time("2022-06-20 05:00:00") +@pytest.mark.integration +def test_operation_circuit_breaker_with_valid_response(pytestconfig): + with patch("gql.client.Client.execute") as mock_gql_client: + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + f = open( + f"{test_resources_dir}/operation_gql_response.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [data] + + config = OperationCircuitBreakerConfig(datahub_host="dummy") + cb = OperationCircuitBreaker(config) + + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.jaffle_shop.customers,PROD)" + ) + assert result is False + + +@freeze_time("2022-06-21 07:00:00") +@pytest.mark.integration +def test_operation_circuit_breaker_with_not_recent_operation(pytestconfig): + with patch("gql.client.Client.execute") as mock_gql_client: + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + f = open( + f"{test_resources_dir}/operation_gql_response.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [data] + + config = OperationCircuitBreakerConfig(datahub_host="dummy") + cb = OperationCircuitBreaker(config) + + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,my_project.jaffle_shop.customers,PROD)" + ) + assert result is True + + +@pytest.mark.integration +def test_assertion_circuit_breaker_with_empty_response(pytestconfig): + with patch("gql.client.Client.execute") as mock_gql_client: + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + f = open( + f"{test_resources_dir}/assertion_gql_empty_response.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [lastUpdatedResponseBeforeLastAssertion, data] + + config = AssertionCircuitBreakerConfig(datahub_host="dummy") + cb = AssertionCircuitBreaker(config) + + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)" + ) + assert result is True + + +@pytest.mark.integration +def test_assertion_circuit_breaker_with_no_error(pytestconfig): + with patch("gql.client.Client.execute") as mock_gql_client: + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + f = open( + f"{test_resources_dir}/assertion_gql_response_with_no_error.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [lastUpdatedResponseBeforeLastAssertion, data] + + config = AssertionCircuitBreakerConfig(datahub_host="dummy") + cb = AssertionCircuitBreaker(config) + + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)" + ) + assert result is False + + +@pytest.mark.integration +def test_assertion_circuit_breaker_updated_at_after_last_assertion(pytestconfig): + with patch("gql.client.Client.execute") as mock_gql_client: + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + f = open( + f"{test_resources_dir}/assertion_gql_response_with_no_error.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [lastUpdatedResponseAfterLastAssertion, data] + + config = AssertionCircuitBreakerConfig(datahub_host="dummy") + cb = AssertionCircuitBreaker(config) + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)" + ) + assert result is True + + +@pytest.mark.integration +def test_assertion_circuit_breaker_assertion_with_active_assertion(pytestconfig): + test_resources_dir = pytestconfig.rootpath / "tests/integration/circuit_breaker" + with patch("gql.client.Client.execute") as mock_gql_client: + f = open( + f"{test_resources_dir}/assertion_gql_response.json", + ) + data = json.load(f) + mock_gql_client.side_effect = [lastUpdatedResponseBeforeLastAssertion, data] + config = AssertionCircuitBreakerConfig(datahub_host="dummy") + cb = AssertionCircuitBreaker(config) + result = cb.is_circuit_breaker_active( + urn="urn:li:dataset:(urn:li:dataPlatform:postgres,postgres1.postgres.public.foo1,PROD)" + ) + assert result is True # add assertion here diff --git a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json index eee87d3605ab30..46dc2df78bcbc6 100644 --- a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json +++ b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json @@ -189,7 +189,12 @@ "sorting_key": "col_Int64", "primary_key": "col_Int64", "sampling_key": "", - "storage_policy": "default" + "storage_policy": "default", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "10", + "total_bytes": "671", + "data_paths": "['/var/lib/clickhouse/store/b1e/b1e5b559-2223-47ac-a34f-4db5cc18fc01/']", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/mv_target_table.sql" }, "externalUrl": null, "name": "mv_target_table", @@ -229,6 +234,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -239,6 +246,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -246,6 +254,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -256,6 +266,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -263,6 +274,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -273,6 +286,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -280,6 +294,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -290,6 +306,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -297,6 +314,8 @@ "jsonPath": null, "nullable": true, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -307,6 +326,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -403,7 +423,12 @@ "sorting_key": "", "primary_key": "", "sampling_key": "", - "storage_policy": "default" + "storage_policy": "default", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "0", + "total_bytes": "0", + "data_paths": "['/var/lib/clickhouse/store/7a7/7a72a81a-cba6-44fa-b0fb-a497d76dee9b/']", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_data_types.sql" }, "externalUrl": null, "name": "test_data_types", @@ -443,6 +468,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -455,6 +482,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -462,6 +490,8 @@ "jsonPath": null, "nullable": false, "description": "https://github.com/ClickHouse/ClickHouse/pull/31072", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -472,6 +502,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -479,6 +510,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -489,6 +522,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -496,6 +530,8 @@ "jsonPath": null, "nullable": false, "description": "this type was added in ClickHouse v21.9", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -506,6 +542,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -513,6 +550,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -523,6 +562,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -530,6 +570,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NullType": {} @@ -540,6 +582,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -547,6 +590,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -557,6 +602,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -564,6 +610,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -574,6 +622,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -581,6 +630,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NullType": {} @@ -591,6 +642,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -598,6 +650,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -608,6 +662,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -615,6 +670,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -625,6 +682,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -632,6 +690,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -642,6 +702,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -649,6 +710,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -659,6 +722,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -666,6 +730,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -676,6 +742,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -683,6 +750,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.EnumType": {} @@ -693,6 +762,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -700,6 +770,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.EnumType": {} @@ -710,6 +782,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -717,6 +790,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.EnumType": {} @@ -727,6 +802,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -734,6 +810,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -744,6 +822,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -751,6 +830,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -761,6 +842,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -768,6 +850,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -778,6 +862,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -785,6 +870,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -795,6 +882,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -802,6 +890,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -812,6 +902,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -819,6 +910,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -829,6 +922,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -836,6 +930,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -846,6 +942,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -853,6 +950,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -863,6 +962,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -870,6 +970,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -880,6 +982,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -887,6 +990,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -897,6 +1002,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -904,6 +1010,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -914,6 +1022,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -921,6 +1030,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.MapType": { @@ -934,6 +1045,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -941,6 +1053,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -951,6 +1065,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -958,6 +1073,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.UnionType": { @@ -970,6 +1087,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -977,6 +1095,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -987,6 +1107,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -994,6 +1115,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1004,6 +1127,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1011,6 +1135,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1021,6 +1147,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1028,6 +1155,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1038,6 +1167,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1045,6 +1175,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1055,6 +1187,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1062,6 +1195,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1072,6 +1207,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1079,6 +1215,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1089,6 +1227,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1204,7 +1343,12 @@ "sorting_key": "", "primary_key": "", "sampling_key": "", - "storage_policy": "" + "storage_policy": "", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "None", + "total_bytes": "None", + "data_paths": "[]", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_dict.sql" }, "externalUrl": null, "name": "test_dict", @@ -1244,6 +1388,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1254,6 +1400,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1261,6 +1408,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1271,6 +1420,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1367,7 +1517,12 @@ "sorting_key": "", "primary_key": "", "sampling_key": "", - "storage_policy": "default" + "storage_policy": "default", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "0", + "total_bytes": "0", + "data_paths": "['/var/lib/clickhouse/store/8c6/8c60ab70-ae8a-46c9-820d-7aa9685c1aac/']", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_nested_data_types.sql" }, "externalUrl": null, "name": "test_nested_data_types", @@ -1407,6 +1562,8 @@ "jsonPath": null, "nullable": false, "description": "this is a comment", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -1419,6 +1576,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1426,6 +1584,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1436,6 +1596,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1443,6 +1604,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NullType": {} @@ -1453,6 +1616,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1460,6 +1624,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NullType": {} @@ -1470,6 +1636,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1477,6 +1644,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -1489,6 +1658,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1496,6 +1666,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -1508,6 +1680,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1515,6 +1688,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -1527,6 +1702,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1534,6 +1710,8 @@ "jsonPath": null, "nullable": true, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1544,6 +1722,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1551,6 +1730,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -1563,6 +1744,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1570,6 +1752,8 @@ "jsonPath": null, "nullable": true, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1580,6 +1764,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1696,6 +1881,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "None", + "total_bytes": "None", + "data_paths": "['/var/lib/clickhouse/store/b1e/b1e5b559-2223-47ac-a34f-4db5cc18fc01/']", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/mv_with_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1737,6 +1927,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1747,6 +1939,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1754,6 +1947,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1764,6 +1959,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1771,6 +1967,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1781,6 +1979,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1788,6 +1987,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1798,6 +1999,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1805,6 +2007,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1815,6 +2019,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1950,6 +2155,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "0", + "total_bytes": "0", + "data_paths": "['/var/lib/clickhouse/store/42a/42a1154e-98ea-4b06-bd9d-16837f11a181/']", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/mv_without_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1991,6 +2201,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -2003,6 +2215,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2010,6 +2223,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2020,6 +2235,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2027,6 +2243,8 @@ "jsonPath": null, "nullable": true, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2037,6 +2255,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2044,6 +2263,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -2056,6 +2277,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2063,6 +2285,8 @@ "jsonPath": null, "nullable": true, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2073,6 +2297,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2208,6 +2433,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", + "metadata_modification_time": "2022-08-08 19:48:20", + "total_rows": "None", + "total_bytes": "None", + "data_paths": "[]", + "metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_view.sql", "view_definition": "", "is_view": "True" }, @@ -2249,6 +2479,8 @@ "jsonPath": null, "nullable": false, "description": "", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2259,6 +2491,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/clickhouse/docker-compose.yml b/metadata-ingestion/tests/integration/clickhouse/docker-compose.yml index 003c7bd1b730ed..756de122a7b61a 100644 --- a/metadata-ingestion/tests/integration/clickhouse/docker-compose.yml +++ b/metadata-ingestion/tests/integration/clickhouse/docker-compose.yml @@ -1,7 +1,7 @@ version: "2" services: clickhouse: - image: yandex/clickhouse-server:22.1 + image: clickhouse/clickhouse-server:22.5.2 container_name: "testclickhouse" environment: CLICKHOUSE_USER: clickhouseuser diff --git a/metadata-ingestion/tests/integration/clickhouse/test_clickhouse.py b/metadata-ingestion/tests/integration/clickhouse/test_clickhouse.py index 28acd786a0f8a6..b340eab43750a9 100644 --- a/metadata-ingestion/tests/integration/clickhouse/test_clickhouse.py +++ b/metadata-ingestion/tests/integration/clickhouse/test_clickhouse.py @@ -1,3 +1,5 @@ +from typing import List + import pytest from freezegun import freeze_time @@ -23,10 +25,16 @@ def test_clickhouse_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_t run_datahub_cmd( ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path ) - + # These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to these. + ignore_paths: List[str] = [ + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['metadata_modification_time'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['data_paths'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['metadata_path'\]", + ] # Verify the output. mce_helpers.check_golden_file( pytestconfig, + ignore_paths=ignore_paths, output_path=tmp_path / "clickhouse_mces.json", golden_path=test_resources_dir / "clickhouse_mces_golden.json", ) diff --git a/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json b/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json new file mode 100644 index 00000000000000..00b77a9240b23a --- /dev/null +++ b/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_golden.json @@ -0,0 +1,116 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "value": "{\"terms\": [{\"urn\": \"urn:li:glossaryTerm:CustomerAccount\"}], \"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:ingestion\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "value": "{\"tags\": [{\"tag\": \"urn:li:tag:Legacy\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:datahub\", \"type\": \"TECHNICAL_OWNER\"}, {\"owner\": \"urn:li:corpuser:jdoe\", \"type\": \"TECHNICAL_OWNER\"}], \"lastModified\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:ingestion\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:Engineering\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "editableDatasetProperties", + "aspect": { + "value": "{\"created\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:ingestion\"}, \"lastModified\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:ingestion\"}, \"description\": \"new description\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "editableSchemaMetadata", + "aspect": { + "value": "{\"created\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:ingestion\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"editableSchemaFieldInfo\": [{\"fieldPath\": \"field_foo\", \"description\": \"field_foo!\", \"glossaryTerms\": {\"terms\": [{\"urn\": \"urn:li:glossaryTerm:AccountBalance\"}], \"auditStamp\": {\"time\": 1643871600000, \"actor\": \"urn:li:corpuser:ingestion\"}}}, {\"fieldPath\": \"field_bar\", \"description\": \"field_bar?\", \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Legacy\"}]}}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test-csv-enricher", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_test_data.csv b/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_test_data.csv new file mode 100644 index 00000000000000..d92dcbf7172fca --- /dev/null +++ b/metadata-ingestion/tests/integration/csv-enricher/csv_enricher_test_data.csv @@ -0,0 +1,4 @@ +resource,subresource,glossary_terms,tags,owners,ownership_type,description,domain +"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",,[urn:li:glossaryTerm:CustomerAccount],[urn:li:tag:Legacy],[urn:li:corpuser:datahub|urn:li:corpuser:jdoe],TECHNICAL_OWNER,new description,urn:li:domain:Engineering +"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",field_foo,[urn:li:glossaryTerm:AccountBalance],,,,field_foo! +"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)",field_bar,,[urn:li:tag:Legacy],,,field_bar? \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/csv-enricher/test_csv_enricher.py b/metadata-ingestion/tests/integration/csv-enricher/test_csv_enricher.py new file mode 100644 index 00000000000000..4a447037d1dda2 --- /dev/null +++ b/metadata-ingestion/tests/integration/csv-enricher/test_csv_enricher.py @@ -0,0 +1,57 @@ +import pathlib + +from freezegun import freeze_time + +from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.csv_enricher import CSVEnricherConfig +from tests.test_helpers import mce_helpers + +FROZEN_TIME = "2022-02-03 07:00:00" + + +def test_csv_enricher_config(): + config = CSVEnricherConfig.parse_obj( + dict( + filename="../integration/csv_enricher/csv_enricher_test_data.csv", + write_semantics="OVERRIDE", + delimiter=",", + array_delimiter="|", + ) + ) + assert config + + +@freeze_time(FROZEN_TIME) +def test_csv_enricher_source(pytestconfig, tmp_path): + test_resources_dir: pathlib.Path = ( + pytestconfig.rootpath / "tests/integration/csv-enricher" + ) + + pipeline = Pipeline.create( + { + "run_id": "test-csv-enricher", + "source": { + "type": "csv-enricher", + "config": { + "filename": f"{test_resources_dir}/csv_enricher_test_data.csv", + "write_semantics": "OVERRIDE", + "delimiter": ",", + "array_delimiter": "|", + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/csv_enricher.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "csv_enricher.json", + golden_path=test_resources_dir / "csv_enricher_golden.json", + ) diff --git a/metadata-ingestion/tests/integration/data_lake/test_data_lake.py b/metadata-ingestion/tests/integration/data_lake/test_data_lake.py index d49b950fb8a9f5..4d51d5d16ff1ab 100644 --- a/metadata-ingestion/tests/integration/data_lake/test_data_lake.py +++ b/metadata-ingestion/tests/integration/data_lake/test_data_lake.py @@ -6,7 +6,7 @@ FROZEN_TIME = "2020-04-14 07:00:00" -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_data_lake_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/data_lake/" diff --git a/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json index da6389769a5994..ee5ad6d0b4f5c9 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json @@ -33,6 +33,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -79,6 +80,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -89,6 +92,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -96,6 +100,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -106,6 +112,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -113,6 +120,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -123,6 +132,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -199,6 +209,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -254,7 +265,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -263,7 +274,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -272,7 +283,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -332,6 +343,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -378,6 +390,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -388,6 +402,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -395,6 +410,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -405,6 +422,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -412,6 +430,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -422,6 +442,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -429,6 +450,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -439,6 +462,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -525,6 +549,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -571,6 +596,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -581,6 +608,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -588,6 +616,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -598,6 +628,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -605,6 +636,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -615,6 +648,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -622,6 +656,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -632,6 +668,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -639,6 +676,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -649,6 +688,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -656,6 +696,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -666,6 +708,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -683,7 +726,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -692,7 +735,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" }, { @@ -701,7 +744,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -710,7 +753,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -719,7 +762,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -728,7 +771,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } ], @@ -787,6 +830,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -833,6 +877,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -843,6 +889,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -850,6 +897,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -860,6 +909,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -867,6 +917,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -877,6 +929,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -884,6 +937,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -894,6 +949,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -901,6 +957,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -911,6 +969,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -918,6 +977,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -928,6 +989,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -935,6 +997,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -945,6 +1009,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -952,6 +1017,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -962,6 +1029,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1031,6 +1099,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -1077,6 +1146,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1087,6 +1158,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1094,6 +1166,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1104,6 +1178,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1111,6 +1186,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1121,6 +1198,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1190,6 +1268,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -1236,6 +1315,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1246,6 +1327,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1253,6 +1335,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1263,6 +1347,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1270,6 +1355,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1280,6 +1367,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1287,6 +1375,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1297,6 +1387,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1369,6 +1460,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -1431,6 +1523,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1441,6 +1535,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1448,6 +1543,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1458,6 +1555,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1465,6 +1563,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1475,6 +1575,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1544,6 +1645,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -1590,6 +1692,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1600,6 +1704,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1607,6 +1712,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1617,6 +1724,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1624,6 +1732,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1634,6 +1744,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1641,6 +1752,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1651,6 +1764,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1658,6 +1772,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1668,6 +1784,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1675,6 +1792,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1685,6 +1804,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1692,6 +1812,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1702,6 +1824,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1709,6 +1832,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1719,6 +1844,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1726,6 +1852,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1736,6 +1864,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1743,6 +1872,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1753,6 +1884,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1822,6 +1954,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -1868,6 +2001,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1878,6 +2013,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1885,6 +2021,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1895,6 +2033,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1902,6 +2041,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1912,6 +2053,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1919,6 +2061,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1929,6 +2073,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1936,6 +2081,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1946,6 +2093,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1953,6 +2101,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1963,6 +2113,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2036,6 +2187,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -2098,6 +2250,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2108,6 +2262,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2115,6 +2270,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2282,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2132,6 +2290,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2142,6 +2302,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2149,6 +2310,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2159,6 +2322,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2166,6 +2330,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2176,6 +2342,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2183,6 +2350,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2193,6 +2362,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2262,6 +2432,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -2308,6 +2479,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2318,6 +2491,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2325,6 +2499,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2335,6 +2511,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2342,6 +2519,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2352,6 +2531,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2359,6 +2539,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2369,6 +2551,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2376,6 +2559,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2386,6 +2571,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2393,6 +2579,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2403,6 +2591,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2472,6 +2661,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -2518,6 +2708,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2528,6 +2720,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2535,6 +2728,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2545,6 +2740,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2552,6 +2748,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2562,6 +2760,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2569,6 +2768,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2579,6 +2780,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2586,6 +2788,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2596,6 +2800,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2603,6 +2808,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2613,6 +2820,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2682,6 +2890,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -2728,6 +2937,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2738,6 +2949,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2745,6 +2957,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2755,6 +2969,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2762,6 +2977,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2772,6 +2989,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2779,6 +2997,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2789,6 +3009,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2796,6 +3017,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2806,6 +3029,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2813,6 +3037,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2823,6 +3049,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2892,6 +3119,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", "manifest_version": "1.0.3", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, @@ -2938,6 +3166,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2948,6 +3178,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2955,6 +3186,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2965,6 +3198,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2972,6 +3206,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2982,6 +3218,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2989,6 +3226,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2999,6 +3238,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3006,6 +3246,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3016,6 +3258,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3023,6 +3266,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3033,6 +3278,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -3174,7 +3420,7 @@ { "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", @@ -3193,7 +3439,7 @@ { "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index da253f26d991e6..605407f2f9727a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -27,6 +27,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -37,6 +38,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -61,7 +63,10 @@ { "owner": "urn:li:corpuser:alice2", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } }, { "owner": "urn:li:corpuser:jdoe.last@gmail.com", @@ -83,16 +88,20 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:has_pii_test" + "tag": "urn:li:tag:dbt:has_pii_test", + "context": null }, { - "tag": "urn:li:tag:dbt:int_meta_property" + "tag": "urn:li:tag:dbt:int_meta_property", + "context": null }, { - "tag": "urn:li:tag:dbt:my_query_tag" + "tag": "urn:li:tag:dbt:my_query_tag", + "context": null }, { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -101,10 +110,12 @@ "com.linkedin.pegasus2avro.common.GlossaryTerms": { "terms": [ { - "urn": "urn:li:glossaryTerm:Finance_test" + "urn": "urn:li:glossaryTerm:Finance_test", + "context": null }, { - "urn": "urn:li:glossaryTerm:double_meta_property" + "urn": "urn:li:glossaryTerm:double_meta_property", + "context": null } ], "auditStamp": { @@ -153,7 +164,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -162,7 +173,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -171,7 +182,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -236,6 +247,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -301,6 +313,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -311,6 +325,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -318,6 +333,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -328,6 +345,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -335,6 +353,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -345,6 +365,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -352,6 +373,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -362,6 +385,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -448,6 +472,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -494,6 +519,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -504,6 +531,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -511,6 +539,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -521,6 +551,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -528,6 +559,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -538,6 +571,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -545,6 +579,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -555,6 +591,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -562,6 +599,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -572,6 +611,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -579,6 +619,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -589,6 +631,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -606,7 +649,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -615,7 +658,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -624,7 +667,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -633,7 +676,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -642,7 +685,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -651,7 +694,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -660,7 +703,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], @@ -720,6 +763,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -766,6 +810,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -776,6 +822,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -783,6 +830,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -793,6 +842,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -800,6 +850,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -810,6 +862,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -889,6 +942,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -911,7 +965,10 @@ { "owner": "urn:li:corpuser:alice1", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } } ], "lastModified": { @@ -951,6 +1008,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -961,6 +1020,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -968,6 +1028,8 @@ "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -978,12 +1040,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -991,6 +1055,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1001,6 +1067,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1008,6 +1075,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1018,6 +1087,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1087,6 +1157,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1133,6 +1204,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1143,6 +1216,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1150,6 +1224,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1160,6 +1236,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1167,6 +1244,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1177,6 +1256,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1184,6 +1264,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1194,6 +1276,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1201,6 +1284,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1211,6 +1296,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1218,6 +1304,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1228,6 +1316,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1235,6 +1324,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1245,6 +1336,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1252,6 +1344,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1262,6 +1356,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1331,6 +1426,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1377,6 +1473,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1387,6 +1485,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1394,6 +1493,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1404,6 +1505,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1411,6 +1513,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1421,6 +1525,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1490,6 +1595,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1536,6 +1642,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1546,6 +1654,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1553,6 +1662,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1563,6 +1674,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1570,6 +1682,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1580,6 +1694,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1587,6 +1702,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1597,6 +1714,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1669,6 +1787,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1691,7 +1810,10 @@ { "owner": "urn:li:corpuser:bob", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } } ], "lastModified": { @@ -1731,6 +1853,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1741,6 +1865,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1748,6 +1873,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1758,6 +1885,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1765,6 +1893,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1775,6 +1905,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1844,6 +1975,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1890,6 +2022,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1900,6 +2034,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1907,6 +2042,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1917,6 +2054,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1924,6 +2062,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1934,6 +2074,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1941,6 +2082,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1951,6 +2094,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1958,6 +2102,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1968,6 +2114,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1975,6 +2122,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1985,6 +2134,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1992,6 +2142,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2002,6 +2154,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2009,6 +2162,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2019,6 +2174,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2026,6 +2182,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2036,6 +2194,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2043,6 +2202,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2053,6 +2214,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2122,6 +2284,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2168,6 +2331,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2178,6 +2343,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2185,6 +2351,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2195,6 +2363,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2202,6 +2371,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2212,6 +2383,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2219,6 +2391,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2229,6 +2403,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2236,6 +2411,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2246,6 +2423,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2253,6 +2431,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2263,6 +2443,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2336,6 +2517,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2358,7 +2540,10 @@ { "owner": "urn:li:corpuser:charles", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } } ], "lastModified": { @@ -2398,6 +2583,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2408,6 +2595,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2415,6 +2603,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2425,6 +2615,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2432,6 +2623,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2442,6 +2635,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2449,6 +2643,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2459,6 +2655,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2466,6 +2663,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2476,6 +2675,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2483,6 +2683,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2493,6 +2695,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2562,6 +2765,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2608,6 +2812,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2618,6 +2824,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2625,6 +2832,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2635,6 +2844,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2642,6 +2852,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2652,6 +2864,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2659,6 +2872,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2669,6 +2884,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2676,6 +2892,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2686,6 +2904,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2693,6 +2912,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2703,6 +2924,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2772,6 +2994,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2818,6 +3041,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2828,6 +3053,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2835,6 +3061,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2845,6 +3073,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2852,6 +3081,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2862,6 +3093,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2869,6 +3101,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2879,6 +3113,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2886,6 +3121,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2896,6 +3133,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2903,6 +3141,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2913,6 +3153,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2982,6 +3223,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -3028,6 +3270,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3038,6 +3282,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3045,6 +3290,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3055,6 +3302,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3062,6 +3310,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3072,6 +3322,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3079,6 +3330,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3089,6 +3342,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3096,6 +3350,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3106,6 +3362,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3113,6 +3370,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3123,6 +3382,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -3192,6 +3452,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -3238,6 +3499,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3248,6 +3511,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3255,6 +3519,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3265,6 +3531,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3272,6 +3539,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3282,6 +3551,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3289,6 +3559,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3299,6 +3571,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3306,6 +3579,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3316,6 +3591,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3323,6 +3599,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3333,6 +3611,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json index 1d5f4b40664d97..b0bb78e58475b0 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json @@ -27,6 +27,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -37,6 +38,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -75,7 +77,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -119,7 +122,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -128,7 +131,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -137,7 +140,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -202,6 +205,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -248,6 +252,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -258,6 +264,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -265,6 +272,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -275,6 +284,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -282,6 +292,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -292,6 +304,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -299,6 +312,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -309,6 +324,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -395,6 +411,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -441,6 +458,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -451,6 +470,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -458,6 +478,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -468,6 +490,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -475,6 +498,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -485,6 +510,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -492,6 +518,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -502,6 +530,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -509,6 +538,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -519,6 +550,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -526,6 +558,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -536,6 +570,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -553,7 +588,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -562,7 +597,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -571,7 +606,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -580,7 +615,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -589,7 +624,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -598,7 +633,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -607,7 +642,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], @@ -667,6 +702,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -713,6 +749,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -723,6 +761,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -730,6 +769,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -740,6 +781,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -747,6 +789,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -757,6 +801,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -836,6 +881,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -898,6 +944,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -908,6 +956,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -915,6 +964,8 @@ "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -925,12 +976,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -938,6 +991,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -948,6 +1003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -955,6 +1011,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -965,6 +1023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1034,6 +1093,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1080,6 +1140,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1090,6 +1152,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1097,6 +1160,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1107,6 +1172,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1114,6 +1180,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1124,6 +1192,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1131,6 +1200,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1141,6 +1212,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1148,6 +1220,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1158,6 +1232,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1165,6 +1240,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1175,6 +1252,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1182,6 +1260,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1192,6 +1272,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1199,6 +1280,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1209,6 +1292,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1278,6 +1362,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1324,6 +1409,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1334,6 +1421,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1341,6 +1429,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1351,6 +1441,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1358,6 +1449,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1368,6 +1461,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1437,6 +1531,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1483,6 +1578,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1493,6 +1590,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1500,6 +1598,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1510,6 +1610,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1517,6 +1618,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1527,6 +1630,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1534,6 +1638,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1544,6 +1650,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1616,6 +1723,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1678,6 +1786,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1688,6 +1798,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1695,6 +1806,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1705,6 +1818,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1712,6 +1826,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1722,6 +1838,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1791,6 +1908,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1837,6 +1955,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1847,6 +1967,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1854,6 +1975,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1864,6 +1987,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1871,6 +1995,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1881,6 +2007,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1888,6 +2015,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1898,6 +2027,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1905,6 +2035,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1915,6 +2047,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1922,6 +2055,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1932,6 +2067,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1939,6 +2075,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1949,6 +2087,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1956,6 +2095,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1966,6 +2107,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1973,6 +2115,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1983,6 +2127,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1990,6 +2135,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2000,6 +2147,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2069,6 +2217,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2115,6 +2264,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2276,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2132,6 +2284,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2142,6 +2296,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2149,6 +2304,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2159,6 +2316,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2166,6 +2324,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2176,6 +2336,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2183,6 +2344,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2193,6 +2356,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2200,6 +2364,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2210,6 +2376,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2283,6 +2450,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2345,6 +2513,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2355,6 +2525,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2362,6 +2533,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2372,6 +2545,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2379,6 +2553,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2389,6 +2565,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2396,6 +2573,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2406,6 +2585,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2413,6 +2593,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2423,6 +2605,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2430,6 +2613,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2440,6 +2625,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2509,6 +2695,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2555,6 +2742,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2565,6 +2754,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2572,6 +2762,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,6 +2774,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2589,6 +2782,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2599,6 +2794,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2606,6 +2802,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2616,6 +2814,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2623,6 +2822,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2633,6 +2834,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2640,6 +2842,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2650,6 +2854,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2719,6 +2924,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2765,6 +2971,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2775,6 +2983,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2782,6 +2991,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2792,6 +3003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2799,6 +3011,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2809,6 +3023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2816,6 +3031,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2826,6 +3043,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2833,6 +3051,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2843,6 +3063,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2850,6 +3071,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2860,6 +3083,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2929,6 +3153,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2975,6 +3200,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2985,6 +3212,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2992,6 +3220,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3002,6 +3232,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3009,6 +3240,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3019,6 +3252,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3026,6 +3260,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3036,6 +3272,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3043,6 +3280,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3053,6 +3292,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3060,6 +3300,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3070,6 +3312,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -3139,6 +3382,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -3185,6 +3429,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3195,6 +3441,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3202,6 +3449,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3212,6 +3461,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3219,6 +3469,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3229,6 +3481,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3236,6 +3489,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3246,6 +3501,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3253,6 +3509,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3263,6 +3521,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3270,6 +3529,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3280,6 +3541,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json index b23620e7ebc7e5..fa1a1353e79dc9 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json @@ -27,6 +27,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -37,6 +38,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -75,7 +77,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -119,7 +122,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -128,7 +131,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -137,7 +140,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -202,6 +205,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -248,6 +252,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -258,6 +264,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -265,6 +272,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -275,6 +284,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -282,6 +292,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -292,6 +304,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -299,6 +312,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -309,6 +324,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -395,6 +411,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -441,6 +458,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -451,6 +470,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -458,6 +478,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -468,6 +490,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -475,6 +498,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -485,6 +510,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -492,6 +518,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -502,6 +530,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -509,6 +538,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -519,6 +550,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -526,6 +558,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -536,6 +570,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -553,7 +588,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -562,7 +597,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -571,7 +606,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -580,7 +615,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -589,7 +624,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -598,7 +633,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -607,7 +642,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], @@ -667,6 +702,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -713,6 +749,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -723,6 +761,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -730,6 +769,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -740,6 +781,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -747,6 +789,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -757,6 +801,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -836,6 +881,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -898,6 +944,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -908,6 +956,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -915,6 +964,8 @@ "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -925,12 +976,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -938,6 +991,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -948,6 +1003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -955,6 +1011,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -965,6 +1023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1034,6 +1093,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1080,6 +1140,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1090,6 +1152,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1097,6 +1160,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1107,6 +1172,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1114,6 +1180,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1124,6 +1192,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1131,6 +1200,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1141,6 +1212,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1148,6 +1220,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1158,6 +1232,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1165,6 +1240,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1175,6 +1252,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1182,6 +1260,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1192,6 +1272,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1199,6 +1280,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1209,6 +1292,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1278,6 +1362,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1324,6 +1409,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1334,6 +1421,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1341,6 +1429,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1351,6 +1441,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1358,6 +1449,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1368,6 +1461,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1437,6 +1531,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1483,6 +1578,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1493,6 +1590,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1500,6 +1598,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1510,6 +1610,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1517,6 +1618,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1527,6 +1630,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1534,6 +1638,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1544,6 +1650,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1616,6 +1723,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1678,6 +1786,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1688,6 +1798,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1695,6 +1806,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1705,6 +1818,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1712,6 +1826,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1722,6 +1838,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1791,6 +1908,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1837,6 +1955,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1847,6 +1967,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1854,6 +1975,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1864,6 +1987,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1871,6 +1995,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1881,6 +2007,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1888,6 +2015,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1898,6 +2027,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1905,6 +2035,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1915,6 +2047,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1922,6 +2055,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1932,6 +2067,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1939,6 +2075,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1949,6 +2087,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1956,6 +2095,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1966,6 +2107,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1973,6 +2115,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1983,6 +2127,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1990,6 +2135,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2000,6 +2147,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2069,6 +2217,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2115,6 +2264,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2276,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2132,6 +2284,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2142,6 +2296,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2149,6 +2304,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2159,6 +2316,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2166,6 +2324,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2176,6 +2336,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2183,6 +2344,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2193,6 +2356,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2200,6 +2364,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2210,6 +2376,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2283,6 +2450,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2345,6 +2513,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2355,6 +2525,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2362,6 +2533,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2372,6 +2545,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2379,6 +2553,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2389,6 +2565,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2396,6 +2573,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2406,6 +2585,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2413,6 +2593,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2423,6 +2605,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2430,6 +2613,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2440,6 +2625,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2509,6 +2695,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2555,6 +2742,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2565,6 +2754,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2572,6 +2762,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,6 +2774,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2589,6 +2782,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2599,6 +2794,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2606,6 +2802,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2616,6 +2814,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2623,6 +2822,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2633,6 +2834,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2640,6 +2842,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2650,6 +2854,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2719,6 +2924,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2765,6 +2971,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2775,6 +2983,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2782,6 +2991,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2792,6 +3003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2799,6 +3011,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2809,6 +3023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2816,6 +3031,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2826,6 +3043,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2833,6 +3051,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2843,6 +3063,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2850,6 +3071,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2860,6 +3083,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2929,6 +3153,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2975,6 +3200,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2985,6 +3212,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2992,6 +3220,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3002,6 +3232,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3009,6 +3240,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3019,6 +3252,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3026,6 +3260,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3036,6 +3272,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3043,6 +3280,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3053,6 +3292,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3060,6 +3300,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3070,6 +3312,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_manifest.json b/metadata-ingestion/tests/integration/dbt/dbt_manifest.json index 5a4e3a6070d0d9..0b16cd2c7d0475 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_manifest.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_manifest.json @@ -2983,6 +2983,7 @@ "name": "sha256" }, "meta": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": true, @@ -3008,10 +3009,7 @@ "quoting": {}, "schema": null, "tags": [], - "vars": {}, - "meta": { - "owner": "@alice2" - } + "vars": {} }, "database": "pagila", "deferred": false, @@ -3059,7 +3057,9 @@ "city" ] ], - "tags": [ "test_tag" ], + "tags": [ + "test_tag" + ], "unique_id": "model.sample_dbt.customer_details", "unrendered_config": { "materialized": "ephemeral" @@ -3360,7 +3360,9 @@ "meta": {}, "name": "first_name", "quote": null, - "tags": ["column_tag"] + "tags": [ + "column_tag" + ] }, "last_name": { "data_type": null, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json new file mode 100644 index 00000000000000..eee040cc407a92 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json @@ -0,0 +1,3315 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_customers.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_payments.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_orders.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_customers.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_orders.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_payments.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/customers.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for a customer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's first name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's last name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's first order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's most recent order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": false, + "description": "Count of the number of orders a customer has placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/orders.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for an order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "Foreign key to the customers table", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) that the order was placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by credit card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by coupon", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by bank transfer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by gift card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "Total amount (AUD) of the order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"assert_total_payment_amount_is_positive\", \"nativeParameters\": {}, \"logic\": \"-- Refunds have a negative amount, so the total amount should always be >= 0.\\n-- Therefore return records where this isn't true to make the test fail\\nselect\\n order_id,\\n sum(amount) as total_amount\\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\\ngroup by 1\\nhaving not(total_amount >= 0)\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_amount\", \"nativeParameters\": {\"column_name\": \"amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_credit_card_amount\", \"nativeParameters\": {\"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),coupon_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_coupon_amount\", \"nativeParameters\": {\"column_name\": \"coupon_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),bank_transfer_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_bank_transfer_amount\", \"nativeParameters\": {\"column_name\": \"bank_transfer_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),gift_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_gift_card_amount\", \"nativeParameters\": {\"column_name\": \"gift_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131075, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:da743330013b7e3e3707ac6e526ab408\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131077, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131073, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:2887b9c826e0be6296a37833bdc380bd\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131058, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b052a324c05327985f3b579a19ad7579\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565137668, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {\"message\": \"Database Error in test dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2 (models/schema.yml)\\n No matching signature for operator = for argument types: INT64, STRING. Supported signature: ANY = ANY at [46:25]\\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565132560, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:44519aa345bf3ea896179f9f352ae946\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565137668, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:1c217b7587a0cad47a07a09bfe154055\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {\"message\": \"Database Error in test dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0 (models/schema.yml)\\n No matching signature for operator = for argument types: FLOAT64, STRING. Supported signature: ANY = ANY at [36:25]\\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133585, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:bbd78a070092f54313153abec49f6f31\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133595, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:ca065a99637630468f688717590beeab\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133591, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:52d06197762e3608d94609e96f03a0a7\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134031, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134482, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134485, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134493, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134966, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135368, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135377, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:c1eebc71f36690e4523adca30314e927\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135510, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135510, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135836, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136269, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136230, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:54bac90e6785bdefd8685ebf8814c429\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136395, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:815963e1332b46a203504ba46ebfab24\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136719, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:fac27f352406b941125292413afa8096\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json new file mode 100644 index 00000000000000..1d7b111a4ada40 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json @@ -0,0 +1,3771 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_customers.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_payments.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_orders.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_customers.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_orders.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_payments.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/customers.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for a customer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's first name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's last name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's first order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's most recent order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": false, + "description": "Count of the number of orders a customer has placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/orders.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for an order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "Foreign key to the customers table", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) that the order was placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by credit card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by coupon", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by bank transfer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by gift card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "Total amount (AUD) of the order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"assert_total_payment_amount_is_positive\", \"nativeParameters\": {}, \"logic\": \"-- Refunds have a negative amount, so the total amount should always be >= 0.\\n-- Therefore return records where this isn't true to make the test fail\\nselect\\n order_id,\\n sum(amount) as total_amount\\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\\ngroup by 1\\nhaving not(total_amount >= 0)\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.assert_total_payment_amount_is_positive,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.unique_stg_customers_customer_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_stg_customers_customer_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.unique_stg_orders_order_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_stg_orders_order_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.unique_stg_payments_payment_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_stg_payments_payment_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.unique_customers_customer_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_customers_customer_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.dbt_expectations_expect_column_24b3791150378f1941309a295b2320de,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.unique_orders_order_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_order_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_customer_id,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.relationships_orders_customer_id__customer_id__ref_customers_,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_amount\", \"nativeParameters\": {\"column_name\": \"amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_amount,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_credit_card_amount\", \"nativeParameters\": {\"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_credit_card_amount,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.dbt_expectations_expect_column_fdf581b1071168614662824120d65b90,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),coupon_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_coupon_amount\", \"nativeParameters\": {\"column_name\": \"coupon_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_coupon_amount,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),bank_transfer_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_bank_transfer_amount\", \"nativeParameters\": {\"column_name\": \"bank_transfer_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_bank_transfer_amount,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),gift_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_gift_card_amount\", \"nativeParameters\": {\"column_name\": \"gift_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop_dbt_test__audit.not_null_orders_gift_card_amount,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131075, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:da743330013b7e3e3707ac6e526ab408\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131077, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131073, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:2887b9c826e0be6296a37833bdc380bd\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131058, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b052a324c05327985f3b579a19ad7579\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565137668, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {\"message\": \"Database Error in test dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2 (models/schema.yml)\\n No matching signature for operator = for argument types: INT64, STRING. Supported signature: ANY = ANY at [46:25]\\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565132560, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:44519aa345bf3ea896179f9f352ae946\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565137668, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:1c217b7587a0cad47a07a09bfe154055\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {\"message\": \"Database Error in test dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0 (models/schema.yml)\\n No matching signature for operator = for argument types: FLOAT64, STRING. Supported signature: ANY = ANY at [36:25]\\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133585, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:bbd78a070092f54313153abec49f6f31\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133595, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:ca065a99637630468f688717590beeab\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133591, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:52d06197762e3608d94609e96f03a0a7\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134031, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134482, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134485, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134493, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134966, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135368, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135377, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:c1eebc71f36690e4523adca30314e927\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135510, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135510, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135836, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136269, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136230, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:54bac90e6785bdefd8685ebf8814c429\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136395, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:815963e1332b46a203504ba46ebfab24\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136719, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:fac27f352406b941125292413afa8096\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-2022_02_03-07_00_00", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index c8d94f8ff32b91..54f4495f16b571 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -37,6 +37,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -75,7 +76,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -119,7 +121,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -128,7 +130,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -137,7 +139,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -202,6 +204,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -248,6 +251,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -258,6 +263,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -265,6 +271,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -275,6 +283,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -282,6 +291,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -292,6 +303,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -299,6 +311,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -309,6 +323,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -395,6 +410,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -441,6 +457,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -451,6 +469,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -458,6 +477,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -468,6 +489,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -475,6 +497,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -485,6 +509,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -492,6 +517,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -502,6 +529,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -509,6 +537,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -519,6 +549,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -526,6 +557,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -536,6 +569,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -553,7 +587,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -562,7 +596,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -571,7 +605,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -580,7 +614,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -589,7 +623,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -598,7 +632,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -607,7 +641,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], @@ -667,6 +701,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -713,6 +748,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -723,6 +760,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -730,6 +768,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -740,6 +780,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -747,6 +788,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -757,6 +800,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -836,6 +880,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -898,6 +943,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -908,6 +955,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -915,6 +963,8 @@ "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -925,12 +975,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -938,6 +990,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -948,6 +1002,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -955,6 +1010,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -965,6 +1022,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1034,6 +1092,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1080,6 +1139,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1090,6 +1151,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1097,6 +1159,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1107,6 +1171,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1114,6 +1179,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1124,6 +1191,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1131,6 +1199,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1141,6 +1211,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1148,6 +1219,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1158,6 +1231,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1165,6 +1239,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1175,6 +1251,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1182,6 +1259,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1192,6 +1271,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1199,6 +1279,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1209,6 +1291,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1278,6 +1361,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1324,6 +1408,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1334,6 +1420,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1341,6 +1428,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1351,6 +1440,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1358,6 +1448,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1368,6 +1460,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1437,6 +1530,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1483,6 +1577,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1493,6 +1589,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1500,6 +1597,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1510,6 +1609,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1517,6 +1617,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1527,6 +1629,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1534,6 +1637,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1544,6 +1649,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1616,6 +1722,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1678,6 +1785,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1688,6 +1797,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1695,6 +1805,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1705,6 +1817,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1712,6 +1825,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1722,6 +1837,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1791,6 +1907,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1837,6 +1954,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1847,6 +1966,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1854,6 +1974,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1864,6 +1986,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1871,6 +1994,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1881,6 +2006,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1888,6 +2014,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1898,6 +2026,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1905,6 +2034,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1915,6 +2046,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1922,6 +2054,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1932,6 +2066,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1939,6 +2074,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1949,6 +2086,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1956,6 +2094,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1966,6 +2106,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1973,6 +2114,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1983,6 +2126,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1990,6 +2134,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2000,6 +2146,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2069,6 +2216,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2115,6 +2263,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2275,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2132,6 +2283,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2142,6 +2295,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2149,6 +2303,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2159,6 +2315,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2166,6 +2323,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2176,6 +2335,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2183,6 +2343,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2193,6 +2355,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2200,6 +2363,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2210,6 +2375,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2283,6 +2449,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2345,6 +2512,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2355,6 +2524,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2362,6 +2532,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2372,6 +2544,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2379,6 +2552,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2389,6 +2564,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2396,6 +2572,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2406,6 +2584,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2413,6 +2592,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2423,6 +2604,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2430,6 +2612,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2440,6 +2624,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2509,6 +2694,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2555,6 +2741,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2565,6 +2753,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2572,6 +2761,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,6 +2773,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2589,6 +2781,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2599,6 +2793,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2606,6 +2801,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2616,6 +2813,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2623,6 +2821,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2633,6 +2833,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2640,6 +2841,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2650,6 +2853,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2719,6 +2923,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2765,6 +2970,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2775,6 +2982,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2782,6 +2990,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2792,6 +3002,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2799,6 +3010,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2809,6 +3022,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2816,6 +3030,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2826,6 +3042,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2833,6 +3050,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2843,6 +3062,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2850,6 +3070,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2860,6 +3082,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2929,6 +3152,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2975,6 +3199,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2985,6 +3211,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2992,6 +3219,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3002,6 +3231,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3009,6 +3239,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3019,6 +3251,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3026,6 +3259,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3036,6 +3271,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3043,6 +3279,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3053,6 +3291,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3060,6 +3299,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3070,6 +3311,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index 02a40c85d1165d..baeafc2b69896c 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -27,6 +27,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -37,6 +38,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -67,8 +69,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -76,7 +77,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -89,14 +91,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -120,30 +120,27 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -208,6 +205,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -232,14 +230,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -344,8 +340,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" @@ -354,8 +349,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" @@ -417,6 +411,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -441,14 +436,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -593,70 +586,63 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], @@ -716,6 +702,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -740,14 +727,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -832,8 +817,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" @@ -897,6 +881,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -925,8 +910,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -938,14 +922,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1581759273000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -994,7 +976,8 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, @@ -1056,8 +1039,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", "type": "TRANSFORMED" @@ -1111,6 +1093,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1135,14 +1118,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1581759930000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -1327,8 +1308,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" @@ -1382,6 +1362,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1406,14 +1387,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1581759987000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -1498,8 +1477,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", "type": "TRANSFORMED" @@ -1553,6 +1531,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1577,14 +1556,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1581759925000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -1689,8 +1666,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" @@ -1747,6 +1723,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1775,8 +1752,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -1788,14 +1764,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1581759840000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -1880,8 +1854,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", "type": "TRANSFORMED" @@ -1935,6 +1908,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1959,14 +1933,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1581760640000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -2191,8 +2163,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" @@ -2246,6 +2217,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2270,14 +2242,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1580505371996, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -2422,8 +2392,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" @@ -2481,6 +2450,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2509,8 +2479,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -2522,14 +2491,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1582319845996, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -2674,8 +2641,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -2729,6 +2695,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2753,14 +2720,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1584998318996, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -2905,8 +2870,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" @@ -2960,6 +2924,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2984,14 +2949,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1588287228996, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -3136,8 +3099,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" @@ -3191,6 +3153,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -3215,14 +3178,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 1589460269996, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -3367,8 +3328,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" @@ -3422,6 +3382,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -3446,14 +3407,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": -62135596800000, "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -3598,8 +3557,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" @@ -3633,8 +3591,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", "type": "TRANSFORMED" @@ -3668,8 +3625,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" @@ -3703,8 +3659,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json new file mode 100644 index 00000000000000..faa36905975126 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -0,0 +1,3683 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "owner": "@alice2", + "business_owner": "jdoe.last@gmail.com", + "data_governance.team_owner": "Finance", + "has_pii": "True", + "int_property": "1", + "double_property": "2.5", + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transform/customer_details.sql", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "customer_details", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [ + "dbt:test_tag" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice2", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:test_tag", + "context": null + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.customer_details", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "business_owner": "jdoe.last", + "data_governance.team_owner": "Sales", + "has_pii": "False", + "int_property": "2", + "double_property": "3.5", + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "an-aliased-view-for-monthly-billing", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.monthly_billing_with_cust", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "billing_month", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "email", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/base/payments_base.sql", + "catalog_type": "VIEW", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "an-aliased-view-for-payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_base", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized=\"view\",\n alias='an-aliased-view-for-payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payments_by_customer_by_month", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_by_customer_by_month", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "billing_month", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n billing_month,\n customer_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in dev", + "some_other_property": "test 1", + "owner": "@alice1", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "actor", + "qualifiedName": null, + "description": "description for actor table from dbt", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice1", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.actor", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759273000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "actor_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:column_tag", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "description for last_name from dbt", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": "description for last_update from dbt", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.actor,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "address", + "qualifiedName": null, + "description": "a user's address", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.address", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759930000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "address", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address2", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "city_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "district", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "phone", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "postal_code", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.address,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "category", + "qualifiedName": null, + "description": "a user's category", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.category", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759987000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "category_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.category,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "city", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.city", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759925000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "city", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "city_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "country_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.city,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "country", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@bob", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.country", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581759840000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "country", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "country_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.country,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "customer", + "qualifiedName": null, + "description": "description for customer table from dbt", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.customer", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1581760640000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activebool", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "create_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "email", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "store_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.customer,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payment_p2020_01", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_01", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1580505371996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "an_array_property": "['alpha', 'beta', 'charlie']", + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payment_p2020_02", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@charles", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_02", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1582319845996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payment_p2020_03", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_03", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1584998318996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payment_p2020_04", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_04", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1588287228996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payment_p2020_05", + "qualifiedName": null, + "description": "a payment", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_05", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1589460269996, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "externalUrl": null, + "name": "payment_p2020_06", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_06", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": -62135596800000, + "actor": "urn:li:corpuser:dbt_executor", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-target-platform-instance", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json index 02a782a66583e6..4b78fa9021c73d 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json @@ -8,6 +8,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -18,6 +19,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -56,7 +58,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -157,6 +160,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -203,6 +207,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -213,6 +219,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -220,6 +227,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -230,6 +239,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -237,6 +247,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -247,6 +259,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -254,6 +267,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -264,6 +279,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -324,6 +340,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -370,6 +387,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -380,6 +399,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -387,6 +407,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -397,6 +419,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -404,6 +427,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -414,6 +439,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -421,6 +447,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -431,6 +459,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -438,6 +467,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -448,6 +479,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -455,6 +487,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -465,6 +499,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -570,6 +605,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -616,6 +652,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -626,6 +664,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -633,6 +672,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -643,6 +684,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -650,6 +692,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -660,6 +704,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -713,6 +758,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -775,6 +821,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -785,6 +833,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -792,6 +841,8 @@ "jsonPath": null, "nullable": false, "description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -802,12 +853,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -815,6 +868,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -825,6 +880,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -832,6 +888,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -842,6 +900,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -876,6 +935,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -922,6 +982,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -932,6 +994,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -939,6 +1002,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -949,6 +1014,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -956,6 +1022,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -966,6 +1034,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -973,6 +1042,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -983,6 +1054,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -990,6 +1062,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1000,6 +1074,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1007,6 +1082,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1017,6 +1094,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1024,6 +1102,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1034,6 +1114,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1041,6 +1122,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1051,6 +1134,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1085,6 +1169,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1131,6 +1216,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1141,6 +1228,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1148,6 +1236,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1158,6 +1248,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1165,6 +1256,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1175,6 +1268,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1209,6 +1303,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1255,6 +1350,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1265,6 +1362,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1272,6 +1370,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1282,6 +1382,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1289,6 +1390,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1299,6 +1402,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1306,6 +1410,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1316,6 +1422,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1353,6 +1460,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1415,6 +1523,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1425,6 +1535,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1432,6 +1543,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1442,6 +1555,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1449,6 +1563,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1459,6 +1575,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1493,6 +1610,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1539,6 +1657,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1549,6 +1669,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1556,6 +1677,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1566,6 +1689,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1573,6 +1697,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1583,6 +1709,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1590,6 +1717,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1600,6 +1729,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1607,6 +1737,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1617,6 +1749,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1624,6 +1757,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1634,6 +1769,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1641,6 +1777,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1651,6 +1789,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1658,6 +1797,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1668,6 +1809,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1675,6 +1817,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1685,6 +1829,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1692,6 +1837,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1702,6 +1849,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1736,6 +1884,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1782,6 +1931,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1792,6 +1943,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1799,6 +1951,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1809,6 +1963,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1816,6 +1971,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1826,6 +1983,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1833,6 +1991,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1843,6 +2003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1850,6 +2011,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1860,6 +2023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1867,6 +2031,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1877,6 +2043,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1915,6 +2082,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1977,6 +2145,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1987,6 +2157,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1994,6 +2165,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2004,6 +2177,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2011,6 +2185,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2021,6 +2197,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2028,6 +2205,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2038,6 +2217,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2045,6 +2225,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2055,6 +2237,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2062,6 +2245,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2072,6 +2257,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2106,6 +2292,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2152,6 +2339,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2162,6 +2351,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2169,6 +2359,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2179,6 +2371,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2186,6 +2379,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2196,6 +2391,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2203,6 +2399,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2213,6 +2411,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2220,6 +2419,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2230,6 +2431,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2237,6 +2439,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2247,6 +2451,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2281,6 +2486,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2327,6 +2533,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2337,6 +2545,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2344,6 +2553,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2354,6 +2565,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2361,6 +2573,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2371,6 +2585,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2378,6 +2593,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2388,6 +2605,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2395,6 +2613,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2405,6 +2625,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2412,6 +2633,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2422,6 +2645,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2456,6 +2680,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2502,6 +2727,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2512,6 +2739,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2519,6 +2747,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2529,6 +2759,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2536,6 +2767,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2546,6 +2779,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2553,6 +2787,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2563,6 +2799,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2570,6 +2807,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2580,6 +2819,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2587,6 +2827,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2597,6 +2839,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2631,6 +2874,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2677,6 +2921,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2687,6 +2933,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2694,6 +2941,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2704,6 +2953,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2711,6 +2961,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2721,6 +2973,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2728,6 +2981,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2738,6 +2993,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2745,6 +3001,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2755,6 +3013,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2762,6 +3021,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2772,6 +3033,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json index 84c1f41bd6e1ff..36534c8b0b7587 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json @@ -8,6 +8,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -18,6 +19,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -42,7 +44,10 @@ { "owner": "urn:li:corpuser:alice2", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } }, { "owner": "urn:li:corpuser:jdoe.last@gmail.com", @@ -64,16 +69,20 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:has_pii_test" + "tag": "urn:li:tag:dbt:has_pii_test", + "context": null }, { - "tag": "urn:li:tag:dbt:int_meta_property" + "tag": "urn:li:tag:dbt:int_meta_property", + "context": null }, { - "tag": "urn:li:tag:dbt:my_query_tag" + "tag": "urn:li:tag:dbt:my_query_tag", + "context": null }, { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -82,10 +91,12 @@ "com.linkedin.pegasus2avro.common.GlossaryTerms": { "terms": [ { - "urn": "urn:li:glossaryTerm:Finance_test" + "urn": "urn:li:glossaryTerm:Finance_test", + "context": null }, { - "urn": "urn:li:glossaryTerm:double_meta_property" + "urn": "urn:li:glossaryTerm:double_meta_property", + "context": null } ], "auditStamp": { @@ -191,6 +202,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -256,6 +268,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -266,6 +280,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -273,6 +288,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -283,6 +300,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -290,6 +308,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -300,6 +320,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -307,6 +328,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -317,6 +340,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -377,6 +401,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -423,6 +448,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -433,6 +460,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -440,6 +468,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -450,6 +480,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -457,6 +488,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -467,6 +500,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -474,6 +508,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -484,6 +520,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -491,6 +528,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -501,6 +540,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -508,6 +548,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -518,6 +560,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -623,6 +666,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -669,6 +713,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -679,6 +725,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -686,6 +733,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -696,6 +745,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -703,6 +753,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -713,6 +765,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -766,6 +819,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -788,7 +842,10 @@ { "owner": "urn:li:corpuser:alice1", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } } ], "lastModified": { @@ -828,6 +885,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -838,6 +897,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -845,6 +905,8 @@ "jsonPath": null, "nullable": false, "description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -855,12 +917,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -868,6 +932,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -878,6 +944,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -885,6 +952,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -895,6 +964,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -929,6 +999,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -975,6 +1046,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -985,6 +1058,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -992,6 +1066,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1002,6 +1078,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1009,6 +1086,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1019,6 +1098,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1026,6 +1106,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1036,6 +1118,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1043,6 +1126,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1053,6 +1138,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1060,6 +1146,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1070,6 +1158,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1077,6 +1166,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1087,6 +1178,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1094,6 +1186,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1104,6 +1198,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1138,6 +1233,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1184,6 +1280,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1194,6 +1292,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1201,6 +1300,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1211,6 +1312,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1218,6 +1320,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1228,6 +1332,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1262,6 +1367,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1308,6 +1414,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1318,6 +1426,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1325,6 +1434,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1335,6 +1446,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1342,6 +1454,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1352,6 +1466,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1359,6 +1474,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1369,6 +1486,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1406,6 +1524,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1428,7 +1547,10 @@ { "owner": "urn:li:corpuser:bob", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } } ], "lastModified": { @@ -1468,6 +1590,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1478,6 +1602,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1485,6 +1610,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1495,6 +1622,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1502,6 +1630,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1512,6 +1642,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1546,6 +1677,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1592,6 +1724,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1602,6 +1736,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1609,6 +1744,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1619,6 +1756,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1626,6 +1764,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1636,6 +1776,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1643,6 +1784,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1653,6 +1796,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1660,6 +1804,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1670,6 +1816,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1677,6 +1824,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1687,6 +1836,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1694,6 +1844,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1704,6 +1856,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1711,6 +1864,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1721,6 +1876,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1728,6 +1884,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1738,6 +1896,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1745,6 +1904,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1755,6 +1916,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1789,6 +1951,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1835,6 +1998,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1845,6 +2010,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1852,6 +2018,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1862,6 +2030,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1869,6 +2038,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1879,6 +2050,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1886,6 +2058,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1896,6 +2070,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1903,6 +2078,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1913,6 +2090,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1920,6 +2098,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1930,6 +2110,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1968,6 +2149,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -1990,7 +2172,10 @@ { "owner": "urn:li:corpuser:charles", "type": "DATAOWNER", - "source": null + "source": { + "type": "SOURCE_CONTROL", + "url": null + } } ], "lastModified": { @@ -2030,6 +2215,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2040,6 +2227,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2047,6 +2235,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2057,6 +2247,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2064,6 +2255,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2074,6 +2267,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2081,6 +2275,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2091,6 +2287,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2098,6 +2295,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2108,6 +2307,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2115,6 +2315,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2327,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2159,6 +2362,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2205,6 +2409,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2215,6 +2421,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2222,6 +2429,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2232,6 +2441,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2239,6 +2449,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2249,6 +2461,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2256,6 +2469,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2266,6 +2481,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2273,6 +2489,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2283,6 +2501,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2290,6 +2509,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2300,6 +2521,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2334,6 +2556,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2380,6 +2603,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2390,6 +2615,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2397,6 +2623,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2407,6 +2635,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2414,6 +2643,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2424,6 +2655,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2431,6 +2663,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2441,6 +2675,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2448,6 +2683,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2458,6 +2695,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2465,6 +2703,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2475,6 +2715,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2509,6 +2750,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2555,6 +2797,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2565,6 +2809,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2572,6 +2817,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,6 +2829,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2589,6 +2837,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2599,6 +2849,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2606,6 +2857,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2616,6 +2869,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2623,6 +2877,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2633,6 +2889,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2640,6 +2897,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2650,6 +2909,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2684,6 +2944,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -2730,6 +2991,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2740,6 +3003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2747,6 +3011,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2757,6 +3023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2764,6 +3031,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2774,6 +3043,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2781,6 +3051,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2791,6 +3063,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2798,6 +3071,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2808,6 +3083,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2815,6 +3091,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2825,6 +3103,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json index 8655765b31942d..04e0cacfb313e5 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json @@ -8,6 +8,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -18,6 +19,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -56,7 +58,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -127,6 +130,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -195,6 +199,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -308,6 +313,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -369,6 +375,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -427,6 +434,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -469,6 +477,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -511,6 +520,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -556,6 +566,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -614,6 +625,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -656,6 +668,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -702,6 +715,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -760,6 +774,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -802,6 +817,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -844,6 +860,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -886,6 +903,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json index 53ac30d001afd3..b2af77796ec5f9 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json @@ -8,6 +8,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "owner": "@alice2", "business_owner": "jdoe.last@gmail.com", "data_governance.team_owner": "Finance", "has_pii": "True", @@ -18,6 +19,7 @@ "dbt_file_path": "models/transform/customer_details.sql", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -56,7 +58,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -127,6 +130,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -195,6 +199,7 @@ "catalog_type": "VIEW", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -308,6 +313,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -369,6 +375,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -427,6 +434,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -469,6 +477,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -511,6 +520,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -556,6 +566,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -614,6 +625,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -656,6 +668,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -702,6 +715,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -760,6 +774,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -802,6 +817,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, @@ -844,6 +860,7 @@ "catalog_type": "BASE TABLE", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", "manifest_version": "0.19.1", + "manifest_adapter": "postgres", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, diff --git a/metadata-ingestion/tests/integration/dbt/jaffle_shop_catalog.json b/metadata-ingestion/tests/integration/dbt/jaffle_shop_catalog.json new file mode 100644 index 00000000000000..29bcbebb7c3a12 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/jaffle_shop_catalog.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", "dbt_version": "1.1.0", "generated_at": "2022-06-17T03:53:07.765940Z", "invocation_id": "e82feb8d-648e-44e4-a3c0-0be32533520e", "env": {}}, "nodes": {"seed.jaffle_shop.raw_orders": {"metadata": {"type": "table", "schema": "jaffle_shop", "name": "raw_orders", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"id": {"type": "INT64", "index": 1, "name": "id", "comment": null}, "user_id": {"type": "INT64", "index": 2, "name": "user_id", "comment": null}, "order_date": {"type": "DATE", "index": 3, "name": "order_date", "comment": null}, "status": {"type": "STRING", "index": 4, "name": "status", "comment": null}}, "stats": {"num_rows": {"id": "num_rows", "label": "# Rows", "value": 99.0, "include": true, "description": "Approximate count of rows in this table"}, "num_bytes": {"id": "num_bytes", "label": "Approximate Size", "value": 3406.0, "include": true, "description": "Approximate size of table as reported by BigQuery"}, "has_stats": {"id": "has_stats", "label": "Has Stats?", "value": true, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "seed.jaffle_shop.raw_orders"}, "seed.jaffle_shop.raw_payments": {"metadata": {"type": "table", "schema": "jaffle_shop", "name": "raw_payments", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"id": {"type": "INT64", "index": 1, "name": "id", "comment": null}, "order_id": {"type": "INT64", "index": 2, "name": "order_id", "comment": null}, "payment_method": {"type": "STRING", "index": 3, "name": "payment_method", "comment": null}, "amount": {"type": "INT64", "index": 4, "name": "amount", "comment": null}}, "stats": {"num_rows": {"id": "num_rows", "label": "# Rows", "value": 113.0, "include": true, "description": "Approximate count of rows in this table"}, "num_bytes": {"id": "num_bytes", "label": "Approximate Size", "value": 4158.0, "include": true, "description": "Approximate size of table as reported by BigQuery"}, "has_stats": {"id": "has_stats", "label": "Has Stats?", "value": true, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "seed.jaffle_shop.raw_payments"}, "model.jaffle_shop.stg_payments": {"metadata": {"type": "view", "schema": "jaffle_shop", "name": "stg_payments", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"payment_id": {"type": "INT64", "index": 1, "name": "payment_id", "comment": null}, "order_id": {"type": "INT64", "index": 2, "name": "order_id", "comment": null}, "payment_method": {"type": "STRING", "index": 3, "name": "payment_method", "comment": null}, "amount": {"type": "FLOAT64", "index": 4, "name": "amount", "comment": null}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "model.jaffle_shop.stg_payments"}, "model.jaffle_shop.orders": {"metadata": {"type": "table", "schema": "jaffle_shop", "name": "orders", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"order_id": {"type": "INT64", "index": 1, "name": "order_id", "comment": null}, "customer_id": {"type": "INT64", "index": 2, "name": "customer_id", "comment": null}, "order_date": {"type": "DATE", "index": 3, "name": "order_date", "comment": null}, "status": {"type": "STRING", "index": 4, "name": "status", "comment": null}, "credit_card_amount": {"type": "FLOAT64", "index": 5, "name": "credit_card_amount", "comment": null}, "coupon_amount": {"type": "FLOAT64", "index": 6, "name": "coupon_amount", "comment": null}, "bank_transfer_amount": {"type": "FLOAT64", "index": 7, "name": "bank_transfer_amount", "comment": null}, "gift_card_amount": {"type": "FLOAT64", "index": 8, "name": "gift_card_amount", "comment": null}, "amount": {"type": "FLOAT64", "index": 9, "name": "amount", "comment": null}}, "stats": {"num_rows": {"id": "num_rows", "label": "# Rows", "value": 99.0, "include": true, "description": "Approximate count of rows in this table"}, "num_bytes": {"id": "num_bytes", "label": "Approximate Size", "value": 7366.0, "include": true, "description": "Approximate size of table as reported by BigQuery"}, "has_stats": {"id": "has_stats", "label": "Has Stats?", "value": true, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "model.jaffle_shop.orders"}, "model.jaffle_shop.stg_orders": {"metadata": {"type": "view", "schema": "jaffle_shop", "name": "stg_orders", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"order_id": {"type": "INT64", "index": 1, "name": "order_id", "comment": null}, "customer_id": {"type": "INT64", "index": 2, "name": "customer_id", "comment": null}, "order_date": {"type": "DATE", "index": 3, "name": "order_date", "comment": null}, "status": {"type": "STRING", "index": 4, "name": "status", "comment": null}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "model.jaffle_shop.stg_orders"}, "model.jaffle_shop.stg_customers": {"metadata": {"type": "view", "schema": "jaffle_shop", "name": "stg_customers", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"customer_id": {"type": "INT64", "index": 1, "name": "customer_id", "comment": null}, "first_name": {"type": "STRING", "index": 2, "name": "first_name", "comment": null}, "last_name": {"type": "STRING", "index": 3, "name": "last_name", "comment": null}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "model.jaffle_shop.stg_customers"}, "model.jaffle_shop.customers": {"metadata": {"type": "table", "schema": "jaffle_shop", "name": "customers", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"customer_id": {"type": "INT64", "index": 1, "name": "customer_id", "comment": null}, "first_name": {"type": "STRING", "index": 2, "name": "first_name", "comment": null}, "last_name": {"type": "STRING", "index": 3, "name": "last_name", "comment": null}, "first_order": {"type": "DATE", "index": 4, "name": "first_order", "comment": null}, "most_recent_order": {"type": "DATE", "index": 5, "name": "most_recent_order", "comment": null}, "number_of_orders": {"type": "INT64", "index": 6, "name": "number_of_orders", "comment": null}, "customer_lifetime_value": {"type": "FLOAT64", "index": 7, "name": "customer_lifetime_value", "comment": null}}, "stats": {"num_rows": {"id": "num_rows", "label": "# Rows", "value": 100.0, "include": true, "description": "Approximate count of rows in this table"}, "num_bytes": {"id": "num_bytes", "label": "Approximate Size", "value": 3970.0, "include": true, "description": "Approximate size of table as reported by BigQuery"}, "has_stats": {"id": "has_stats", "label": "Has Stats?", "value": true, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "model.jaffle_shop.customers"}, "seed.jaffle_shop.raw_customers": {"metadata": {"type": "table", "schema": "jaffle_shop", "name": "raw_customers", "database": "calm-pagoda-323403", "comment": null, "owner": null}, "columns": {"id": {"type": "INT64", "index": 1, "name": "id", "comment": null}, "first_name": {"type": "STRING", "index": 2, "name": "first_name", "comment": null}, "last_name": {"type": "STRING", "index": 3, "name": "last_name", "comment": null}}, "stats": {"num_rows": {"id": "num_rows", "label": "# Rows", "value": 100.0, "include": true, "description": "Approximate count of rows in this table"}, "num_bytes": {"id": "num_bytes", "label": "Approximate Size", "value": 1986.0, "include": true, "description": "Approximate size of table as reported by BigQuery"}, "has_stats": {"id": "has_stats", "label": "Has Stats?", "value": true, "include": false, "description": "Indicates whether there are statistics for this table"}}, "unique_id": "seed.jaffle_shop.raw_customers"}}, "sources": {}, "errors": null} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/jaffle_shop_manifest.json b/metadata-ingestion/tests/integration/dbt/jaffle_shop_manifest.json new file mode 100644 index 00000000000000..09399d60073f69 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/jaffle_shop_manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v5.json", "dbt_version": "1.1.0", "generated_at": "2022-06-17T03:53:04.231685Z", "invocation_id": "e82feb8d-648e-44e4-a3c0-0be32533520e", "env": {}, "project_id": "06e5b98c2db46f8a72cc4f66410e9b3b", "user_id": "52e3bce6-a6c2-4618-9907-a0440d04a00b", "send_anonymous_usage_stats": true, "adapter_type": "bigquery"}, "nodes": {"model.jaffle_shop.stg_customers": {"raw_sql": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", "compiled": true, "resource_type": "model", "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_customers"]}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "staging", "stg_customers"], "unique_id": "model.jaffle_shop.stg_customers", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "staging/stg_customers.sql", "original_file_path": "models/staging/stg_customers.sql", "name": "stg_customers", "alias": "stg_customers", "checksum": {"name": "sha256", "checksum": "6f18a29204dad1de6dbb0c288144c4990742e0a1e065c3b2a67b5f98334c22ba"}, "tags": [], "refs": [["raw_customers"]], "sources": [], "description": "", "columns": {"customer_id": {"name": "customer_id", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true}, "patch_path": "jaffle_shop://models/staging/schema.yml", "compiled_path": "target/compiled/jaffle_shop/models/staging/stg_customers.sql", "build_path": null, "deferred": false, "unrendered_config": {"materialized": "view"}, "created_at": 1654986671.659005, "compiled_sql": "with source as (\n select * from `calm-pagoda-323403`.`jaffle_shop`.`raw_customers`\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`stg_customers`"}, "model.jaffle_shop.stg_payments": {"raw_sql": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", "compiled": true, "resource_type": "model", "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_payments"]}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "staging", "stg_payments"], "unique_id": "model.jaffle_shop.stg_payments", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "staging/stg_payments.sql", "original_file_path": "models/staging/stg_payments.sql", "name": "stg_payments", "alias": "stg_payments", "checksum": {"name": "sha256", "checksum": "eb899938258d1fba27fca716a7c334119912a2f9601282026097a7b6ce8cfcd2"}, "tags": [], "refs": [["raw_payments"]], "sources": [], "description": "", "columns": {"payment_id": {"name": "payment_id", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}, "payment_method": {"name": "payment_method", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true}, "patch_path": "jaffle_shop://models/staging/schema.yml", "compiled_path": "target/compiled/jaffle_shop/models/staging/stg_payments.sql", "build_path": null, "deferred": false, "unrendered_config": {"materialized": "view"}, "created_at": 1654986671.660224, "compiled_sql": "with source as (\n select * from `calm-pagoda-323403`.`jaffle_shop`.`raw_payments`\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`stg_payments`"}, "model.jaffle_shop.stg_orders": {"raw_sql": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", "compiled": true, "resource_type": "model", "depends_on": {"macros": [], "nodes": ["seed.jaffle_shop.raw_orders"]}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "view", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "staging", "stg_orders"], "unique_id": "model.jaffle_shop.stg_orders", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "staging/stg_orders.sql", "original_file_path": "models/staging/stg_orders.sql", "name": "stg_orders", "alias": "stg_orders", "checksum": {"name": "sha256", "checksum": "afffa9cbc57e5fd2cf5898ebf571d444a62c9d6d7929d8133d30567fb9a2ce97"}, "tags": [], "refs": [["raw_orders"]], "sources": [], "description": "", "columns": {"order_id": {"name": "order_id", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}, "status": {"name": "status", "description": "", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true}, "patch_path": "jaffle_shop://models/staging/schema.yml", "compiled_path": "target/compiled/jaffle_shop/models/staging/stg_orders.sql", "build_path": null, "deferred": false, "unrendered_config": {"materialized": "view"}, "created_at": 1654986671.659628, "compiled_sql": "with source as (\n select * from `calm-pagoda-323403`.`jaffle_shop`.`raw_orders`\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`stg_orders`"}, "test.jaffle_shop.assert_total_payment_amount_is_positive": {"raw_sql": "-- Refunds have a negative amount, so the total amount should always be >= 0.\n-- Therefore return records where this isn't true to make the test fail\nselect\n order_id,\n sum(amount) as total_amount\nfrom {{ ref('orders' )}}\ngroup by 1\nhaving not(total_amount >= 0)", "compiled": true, "resource_type": "test", "depends_on": {"macros": [], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "assert_total_payment_amount_is_positive"], "unique_id": "test.jaffle_shop.assert_total_payment_amount_is_positive", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "assert_total_payment_amount_is_positive.sql", "original_file_path": "tests/assert_total_payment_amount_is_positive.sql", "name": "assert_total_payment_amount_is_positive", "alias": "assert_total_payment_amount_is_positive", "checksum": {"name": "sha256", "checksum": "3f23335cec2e625080f3d6eef0c5bbefe5d468184d61958d2121e35239b2c3ad"}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/tests/assert_total_payment_amount_is_positive.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.5841331, "compiled_sql": "-- Refunds have a negative amount, so the total amount should always be >= 0.\n-- Therefore return records where this isn't true to make the test fail\nselect\n order_id,\n sum(amount) as total_amount\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\ngroup by 1\nhaving not(total_amount >= 0)", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null}, "seed.jaffle_shop.raw_customers": {"raw_sql": "", "compiled": true, "resource_type": "seed", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "quote_columns": null, "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "raw_customers"], "unique_id": "seed.jaffle_shop.raw_customers", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "raw_customers.csv", "original_file_path": "seeds/raw_customers.csv", "name": "raw_customers", "alias": "raw_customers", "checksum": {"name": "sha256", "checksum": "24579b4b26098d43265376f3c50be8b10faf8e8fd95f5508074f10f76a12671d"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.593128, "compiled_sql": "", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`raw_customers`"}, "seed.jaffle_shop.raw_orders": {"raw_sql": "", "compiled": true, "resource_type": "seed", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "quote_columns": null, "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "raw_orders"], "unique_id": "seed.jaffle_shop.raw_orders", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "raw_orders.csv", "original_file_path": "seeds/raw_orders.csv", "name": "raw_orders", "alias": "raw_orders", "checksum": {"name": "sha256", "checksum": "ee6c68d1639ec2b23a4495ec12475e09b8ed4b61e23ab0411ea7ec76648356f7"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.594195, "compiled_sql": "", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`raw_orders`"}, "seed.jaffle_shop.raw_payments": {"raw_sql": "", "compiled": true, "resource_type": "seed", "depends_on": {"macros": [], "nodes": []}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "seed", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "quote_columns": null, "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "raw_payments"], "unique_id": "seed.jaffle_shop.raw_payments", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "raw_payments.csv", "original_file_path": "seeds/raw_payments.csv", "name": "raw_payments", "alias": "raw_payments", "checksum": {"name": "sha256", "checksum": "03fd407f3135f84456431a923f22fc185a2154079e210c20b690e3ab11687d11"}, "tags": [], "refs": [], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": null, "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.595192, "compiled_sql": "", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`raw_payments`"}, "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": {"raw_sql": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "unique", "kwargs": {"column_name": "customer_id", "model": "{{ get_where_subquery(ref('stg_customers')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_customers"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "unique_stg_customers_customer_id"], "unique_id": "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "unique_stg_customers_customer_id.sql", "original_file_path": "models/staging/schema.yml", "name": "unique_stg_customers_customer_id", "alias": "unique_stg_customers_customer_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_customers"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/unique_stg_customers_customer_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.6606321, "compiled_sql": "\n \n \n\nwith dbt_test__target as (\n\n select customer_id as unique_field\n from `calm-pagoda-323403`.`jaffle_shop`.`stg_customers`\n where customer_id is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.stg_customers"}, "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "customer_id", "model": "{{ get_where_subquery(ref('stg_customers')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_customers"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "not_null_stg_customers_customer_id"], "unique_id": "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_stg_customers_customer_id.sql", "original_file_path": "models/staging/schema.yml", "name": "not_null_stg_customers_customer_id", "alias": "not_null_stg_customers_customer_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_customers"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/not_null_stg_customers_customer_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.661726, "compiled_sql": "\n \n \n\n\n\nselect customer_id\nfrom `calm-pagoda-323403`.`jaffle_shop`.`stg_customers`\nwhere customer_id is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.stg_customers"}, "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": {"raw_sql": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "unique", "kwargs": {"column_name": "order_id", "model": "{{ get_where_subquery(ref('stg_orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "unique_stg_orders_order_id"], "unique_id": "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "unique_stg_orders_order_id.sql", "original_file_path": "models/staging/schema.yml", "name": "unique_stg_orders_order_id", "alias": "unique_stg_orders_order_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/unique_stg_orders_order_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.662907, "compiled_sql": "\n \n \n\nwith dbt_test__target as (\n\n select order_id as unique_field\n from `calm-pagoda-323403`.`jaffle_shop`.`stg_orders`\n where order_id is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "order_id", "file_key_name": "models.stg_orders"}, "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "order_id", "model": "{{ get_where_subquery(ref('stg_orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "not_null_stg_orders_order_id"], "unique_id": "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_stg_orders_order_id.sql", "original_file_path": "models/staging/schema.yml", "name": "not_null_stg_orders_order_id", "alias": "not_null_stg_orders_order_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/not_null_stg_orders_order_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.663967, "compiled_sql": "\n \n \n\n\n\nselect order_id\nfrom `calm-pagoda-323403`.`jaffle_shop`.`stg_orders`\nwhere order_id is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "order_id", "file_key_name": "models.stg_orders"}, "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": {"raw_sql": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58\") }}", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["placed", "shipped", "completed", "return_pending", "returned"], "column_name": "status", "model": "{{ get_where_subquery(ref('stg_orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_orders"]}, "config": {"enabled": true, "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned"], "unique_id": "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58.sql", "original_file_path": "models/staging/schema.yml", "name": "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned", "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58.sql", "build_path": null, "deferred": false, "unrendered_config": {"alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58"}, "created_at": 1654986671.665036, "compiled_sql": "\n \n \n\nwith all_values as (\n\n select\n status as value_field,\n count(*) as n_records\n\n from `calm-pagoda-323403`.`jaffle_shop`.`stg_orders`\n group by status\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n 'placed','shipped','completed','return_pending','returned'\n)\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "status", "file_key_name": "models.stg_orders"}, "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": {"raw_sql": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "unique", "kwargs": {"column_name": "payment_id", "model": "{{ get_where_subquery(ref('stg_payments')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_payments"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "unique_stg_payments_payment_id"], "unique_id": "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "unique_stg_payments_payment_id.sql", "original_file_path": "models/staging/schema.yml", "name": "unique_stg_payments_payment_id", "alias": "unique_stg_payments_payment_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_payments"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/unique_stg_payments_payment_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.66833, "compiled_sql": "\n \n \n\nwith dbt_test__target as (\n\n select payment_id as unique_field\n from `calm-pagoda-323403`.`jaffle_shop`.`stg_payments`\n where payment_id is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "payment_id", "file_key_name": "models.stg_payments"}, "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "payment_id", "model": "{{ get_where_subquery(ref('stg_payments')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_payments"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "not_null_stg_payments_payment_id"], "unique_id": "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_stg_payments_payment_id.sql", "original_file_path": "models/staging/schema.yml", "name": "not_null_stg_payments_payment_id", "alias": "not_null_stg_payments_payment_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_payments"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/not_null_stg_payments_payment_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1654986671.6695318, "compiled_sql": "\n \n \n\n\n\nselect payment_id\nfrom `calm-pagoda-323403`.`jaffle_shop`.`stg_payments`\nwhere payment_id is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "payment_id", "file_key_name": "models.stg_payments"}, "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": {"raw_sql": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef\") }}", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["credit_card", "coupon", "bank_transfer", "gift_card"], "column_name": "payment_method", "model": "{{ get_where_subquery(ref('stg_payments')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.stg_payments"]}, "config": {"enabled": true, "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "staging", "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card"], "unique_id": "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef.sql", "original_file_path": "models/staging/schema.yml", "name": "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card", "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["stg_payments"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/staging/schema.yml/accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef.sql", "build_path": null, "deferred": false, "unrendered_config": {"alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef"}, "created_at": 1654986671.67061, "compiled_sql": "\n \n \n\nwith all_values as (\n\n select\n payment_method as value_field,\n count(*) as n_records\n\n from `calm-pagoda-323403`.`jaffle_shop`.`stg_payments`\n group by payment_method\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n 'credit_card','coupon','bank_transfer','gift_card'\n)\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "payment_method", "file_key_name": "models.stg_payments"}, "model.jaffle_shop.customers": {"raw_sql": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", "compiled": true, "resource_type": "model", "depends_on": {"macros": [], "nodes": ["model.jaffle_shop.stg_customers", "model.jaffle_shop.stg_orders", "model.jaffle_shop.stg_payments"]}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "table", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "customers"], "unique_id": "model.jaffle_shop.customers", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "customers.sql", "original_file_path": "models/customers.sql", "name": "customers", "alias": "customers", "checksum": {"name": "sha256", "checksum": "455b90a31f418ae776213ad9932c7cb72d19a5269a8c722bd9f4e44957313ce8"}, "tags": [], "refs": [["stg_customers"], ["stg_orders"], ["stg_payments"]], "sources": [], "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", "columns": {"customer_id": {"name": "customer_id", "description": "This is a unique identifier for a customer", "meta": {}, "data_type": null, "quote": null, "tags": []}, "first_name": {"name": "first_name", "description": "Customer's first name. PII.", "meta": {}, "data_type": null, "quote": null, "tags": []}, "last_name": {"name": "last_name", "description": "Customer's last name. PII.", "meta": {}, "data_type": null, "quote": null, "tags": []}, "first_order": {"name": "first_order", "description": "Date (UTC) of a customer's first order", "meta": {}, "data_type": null, "quote": null, "tags": []}, "most_recent_order": {"name": "most_recent_order", "description": "Date (UTC) of a customer's most recent order", "meta": {}, "data_type": null, "quote": null, "tags": []}, "number_of_orders": {"name": "number_of_orders", "description": "Count of the number of orders a customer has placed", "meta": {}, "data_type": null, "quote": null, "tags": []}, "total_order_amount": {"name": "total_order_amount", "description": "Total value (AUD) of a customer's orders", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true}, "patch_path": "jaffle_shop://models/schema.yml", "compiled_path": "target/compiled/jaffle_shop/models/customers.sql", "build_path": null, "deferred": false, "unrendered_config": {"materialized": "table"}, "created_at": 1655435797.62363, "compiled_sql": "with customers as (\n\n select * from `calm-pagoda-323403`.`jaffle_shop`.`stg_customers`\n\n),\n\norders as (\n\n select * from `calm-pagoda-323403`.`jaffle_shop`.`stg_orders`\n\n),\n\npayments as (\n\n select * from `calm-pagoda-323403`.`jaffle_shop`.`stg_payments`\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`customers`"}, "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": {"raw_sql": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "unique", "kwargs": {"column_name": "customer_id", "model": "{{ get_where_subquery(ref('customers')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.customers"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "unique_customers_customer_id"], "unique_id": "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "unique_customers_customer_id.sql", "original_file_path": "models/schema.yml", "name": "unique_customers_customer_id", "alias": "unique_customers_customer_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["customers"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/unique_customers_customer_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655435797.627693, "compiled_sql": "\n \n \n\nwith dbt_test__target as (\n\n select customer_id as unique_field\n from `calm-pagoda-323403`.`jaffle_shop`.`customers`\n where customer_id is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.customers"}, "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "customer_id", "model": "{{ get_where_subquery(ref('customers')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.customers"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_customers_customer_id"], "unique_id": "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_customers_customer_id.sql", "original_file_path": "models/schema.yml", "name": "not_null_customers_customer_id", "alias": "not_null_customers_customer_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["customers"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_customers_customer_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655435797.629326, "compiled_sql": "\n \n \n\n\n\nselect customer_id\nfrom `calm-pagoda-323403`.`jaffle_shop`.`customers`\nwhere customer_id is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.customers"}, "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False.e67667298f": {"raw_sql": "{{ dbt_expectations.test_expect_column_values_to_be_between(**_dbt_generic_test_kwargs) }}{{ config(alias=\"dbt_expectations_expect_column_24b3791150378f1941309a295b2320de\") }}", "test_metadata": {"name": "expect_column_values_to_be_between", "kwargs": {"min_value": 0, "max_value": 2000000, "row_condition": "customer_id is not null", "strictly": false, "column_name": "customer_id", "model": "{{ get_where_subquery(ref('customers')) }}"}, "namespace": "dbt_expectations"}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt_expectations.test_expect_column_values_to_be_between", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.customers"]}, "config": {"enabled": true, "alias": "dbt_expectations_expect_column_24b3791150378f1941309a295b2320de", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False"], "unique_id": "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False.e67667298f", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "dbt_expectations_expect_column_24b3791150378f1941309a295b2320de.sql", "original_file_path": "models/schema.yml", "name": "dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False", "alias": "dbt_expectations_expect_column_24b3791150378f1941309a295b2320de", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["customers"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_24b3791150378f1941309a295b2320de.sql", "build_path": null, "deferred": false, "unrendered_config": {"alias": "dbt_expectations_expect_column_24b3791150378f1941309a295b2320de"}, "created_at": 1655435797.630472, "compiled_sql": "\n\n\n\n\n\n\n with grouped_expression as (\n select\n \n \n \n \n( 1=1 and customer_id >= 0 and customer_id <= 2000000\n)\n as expression\n\n\n from `calm-pagoda-323403`.`jaffle_shop`.`customers`\n where\n customer_id is not null\n \n \n\n),\nvalidation_errors as (\n\n select\n *\n from\n grouped_expression\n where\n not(expression = true)\n\n)\n\nselect *\nfrom validation_errors\n\n\n\n\n\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.customers"}, "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2.81450cfcd8": {"raw_sql": "{{ dbt_expectations.test_expect_column_values_to_be_in_set(**_dbt_generic_test_kwargs) }}{{ config(alias=\"dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d\") }}", "test_metadata": {"name": "expect_column_values_to_be_in_set", "kwargs": {"value_set": ["0", "1", "2"], "row_condition": "customer_id is not null", "column_name": "customer_id", "model": "{{ get_where_subquery(ref('customers')) }}"}, "namespace": "dbt_expectations"}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt_expectations.test_expect_column_values_to_be_in_set", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.customers"]}, "config": {"enabled": true, "alias": "dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2"], "unique_id": "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2.81450cfcd8", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql", "original_file_path": "models/schema.yml", "name": "dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2", "alias": "dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["customers"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql", "build_path": null, "deferred": false, "unrendered_config": {"alias": "dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d"}, "created_at": 1655435797.64314, "compiled_sql": "\n\nwith all_values as (\n\n select\n customer_id as value_field\n\n from `calm-pagoda-323403`.`jaffle_shop`.`customers`\n \n where customer_id is not null\n \n\n),\nset_values as (\n\n select\n cast('0' as \n string\n) as value_field\n union all\n select\n cast('1' as \n string\n) as value_field\n union all\n select\n cast('2' as \n string\n) as value_field\n \n \n),\nvalidation_errors as (\n -- values from the model that are not in the set\n select\n v.value_field\n from\n all_values v\n left join\n set_values s on v.value_field = s.value_field\n where\n s.value_field is null\n\n)\n\nselect *\nfrom validation_errors\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.customers"}, "model.jaffle_shop.orders": {"raw_sql": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", "compiled": true, "resource_type": "model", "depends_on": {"macros": [], "nodes": ["model.jaffle_shop.stg_orders", "model.jaffle_shop.stg_payments"]}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "materialized": "table", "persist_docs": {}, "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "post-hook": [], "pre-hook": []}, "database": "calm-pagoda-323403", "schema": "jaffle_shop", "fqn": ["jaffle_shop", "orders"], "unique_id": "model.jaffle_shop.orders", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "orders.sql", "original_file_path": "models/orders.sql", "name": "orders", "alias": "orders", "checksum": {"name": "sha256", "checksum": "53950235d8e29690d259e95ee49bda6a5b7911b44c739b738a646dc6014bcfcd"}, "tags": [], "refs": [["stg_orders"], ["stg_payments"]], "sources": [], "description": "This table has basic information about orders, as well as some derived facts based on payments", "columns": {"order_id": {"name": "order_id", "description": "This is a unique identifier for an order", "meta": {}, "data_type": null, "quote": null, "tags": []}, "customer_id": {"name": "customer_id", "description": "Foreign key to the customers table", "meta": {}, "data_type": null, "quote": null, "tags": []}, "order_date": {"name": "order_date", "description": "Date (UTC) that the order was placed", "meta": {}, "data_type": null, "quote": null, "tags": []}, "status": {"name": "status", "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", "meta": {}, "data_type": null, "quote": null, "tags": []}, "amount": {"name": "amount", "description": "Total amount (AUD) of the order", "meta": {}, "data_type": null, "quote": null, "tags": []}, "credit_card_amount": {"name": "credit_card_amount", "description": "Amount of the order (AUD) paid for by credit card", "meta": {}, "data_type": null, "quote": null, "tags": []}, "coupon_amount": {"name": "coupon_amount", "description": "Amount of the order (AUD) paid for by coupon", "meta": {}, "data_type": null, "quote": null, "tags": []}, "bank_transfer_amount": {"name": "bank_transfer_amount", "description": "Amount of the order (AUD) paid for by bank transfer", "meta": {}, "data_type": null, "quote": null, "tags": []}, "gift_card_amount": {"name": "gift_card_amount", "description": "Amount of the order (AUD) paid for by gift card", "meta": {}, "data_type": null, "quote": null, "tags": []}}, "meta": {}, "docs": {"show": true}, "patch_path": "jaffle_shop://models/schema.yml", "compiled_path": "target/compiled/jaffle_shop/models/orders.sql", "build_path": null, "deferred": false, "unrendered_config": {"materialized": "table"}, "created_at": 1655437984.3231518, "compiled_sql": "\n\nwith orders as (\n\n select * from `calm-pagoda-323403`.`jaffle_shop`.`stg_orders`\n\n),\n\npayments as (\n\n select * from `calm-pagoda-323403`.`jaffle_shop`.`stg_payments`\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n sum(case when payment_method = 'credit_card' then amount else 0 end) as credit_card_amount,\n sum(case when payment_method = 'coupon' then amount else 0 end) as coupon_amount,\n sum(case when payment_method = 'bank_transfer' then amount else 0 end) as bank_transfer_amount,\n sum(case when payment_method = 'gift_card' then amount else 0 end) as gift_card_amount,\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n order_payments.credit_card_amount,\n\n order_payments.coupon_amount,\n\n order_payments.bank_transfer_amount,\n\n order_payments.gift_card_amount,\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": "`calm-pagoda-323403`.`jaffle_shop`.`orders`"}, "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": {"raw_sql": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "unique", "kwargs": {"column_name": "order_id", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "unique_orders_order_id"], "unique_id": "test.jaffle_shop.unique_orders_order_id.fed79b3a6e", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "unique_orders_order_id.sql", "original_file_path": "models/schema.yml", "name": "unique_orders_order_id", "alias": "unique_orders_order_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/unique_orders_order_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.327153, "compiled_sql": "\n \n \n\nwith dbt_test__target as (\n\n select order_id as unique_field\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\n where order_id is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "order_id", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "order_id", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_order_id"], "unique_id": "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_order_id.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_order_id", "alias": "not_null_orders_order_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_order_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.3291478, "compiled_sql": "\n \n \n\n\n\nselect order_id\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere order_id is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "order_id", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "customer_id", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_customer_id"], "unique_id": "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_customer_id.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_customer_id", "alias": "not_null_orders_customer_id", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_customer_id.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.330257, "compiled_sql": "\n \n \n\n\n\nselect customer_id\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere customer_id is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.orders"}, "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": {"raw_sql": "{{ test_relationships(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "relationships", "kwargs": {"to": "ref('customers')", "field": "customer_id", "column_name": "customer_id", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_relationships", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.customers", "model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "relationships_orders_customer_id__customer_id__ref_customers_"], "unique_id": "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "relationships_orders_customer_id__customer_id__ref_customers_.sql", "original_file_path": "models/schema.yml", "name": "relationships_orders_customer_id__customer_id__ref_customers_", "alias": "relationships_orders_customer_id__customer_id__ref_customers_", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["customers"], ["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/relationships_orders_customer_id__customer_id__ref_customers_.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.331372, "compiled_sql": "\n \n \n\nwith child as (\n select customer_id as from_field\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\n where customer_id is not null\n),\n\nparent as (\n select customer_id as to_field\n from `calm-pagoda-323403`.`jaffle_shop`.`customers`\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "customer_id", "file_key_name": "models.orders"}, "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": {"raw_sql": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758\") }}", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["placed", "shipped", "completed", "return_pending", "returned"], "column_name": "status", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "accepted_values_orders_status__placed__shipped__completed__return_pending__returned"], "unique_id": "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758.sql", "original_file_path": "models/schema.yml", "name": "accepted_values_orders_status__placed__shipped__completed__return_pending__returned", "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758.sql", "build_path": null, "deferred": false, "unrendered_config": {"alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758"}, "created_at": 1655437984.33775, "compiled_sql": "\n \n \n\nwith all_values as (\n\n select\n status as value_field,\n count(*) as n_records\n\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\n group by status\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n 'placed','shipped','completed','return_pending','returned'\n)\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "status", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_amount.106140f9fd": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "amount", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_amount"], "unique_id": "test.jaffle_shop.not_null_orders_amount.106140f9fd", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_amount.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_amount", "alias": "not_null_orders_amount", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_amount.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.343115, "compiled_sql": "\n \n \n\n\n\nselect amount\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere amount is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "amount", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "credit_card_amount", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_credit_card_amount"], "unique_id": "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_credit_card_amount.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_credit_card_amount", "alias": "not_null_orders_credit_card_amount", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_credit_card_amount.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.344332, "compiled_sql": "\n \n \n\n\n\nselect credit_card_amount\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere credit_card_amount is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "credit_card_amount", "file_key_name": "models.orders"}, "test.jaffle_shop.dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0.888b06036c": {"raw_sql": "{{ dbt_expectations.test_expect_column_values_to_not_be_in_set(**_dbt_generic_test_kwargs) }}{{ config(alias=\"dbt_expectations_expect_column_fdf581b1071168614662824120d65b90\") }}", "test_metadata": {"name": "expect_column_values_to_not_be_in_set", "kwargs": {"value_set": ["0"], "row_condition": "credit_card_amount is not null", "column_name": "credit_card_amount", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": "dbt_expectations"}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt_expectations.test_expect_column_values_to_not_be_in_set", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": "dbt_expectations_expect_column_fdf581b1071168614662824120d65b90", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0"], "unique_id": "test.jaffle_shop.dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0.888b06036c", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql", "original_file_path": "models/schema.yml", "name": "dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0", "alias": "dbt_expectations_expect_column_fdf581b1071168614662824120d65b90", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql", "build_path": null, "deferred": false, "unrendered_config": {"alias": "dbt_expectations_expect_column_fdf581b1071168614662824120d65b90"}, "created_at": 1655437984.345427, "compiled_sql": "\n\nwith all_values as (\n\n select\n credit_card_amount as value_field\n\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\n \n where credit_card_amount is not null\n \n\n),\nset_values as (\n\n select\n cast('0' as \n string\n) as value_field\n \n \n),\nvalidation_errors as (\n -- values from the model that match the set\n select\n v.value_field\n from\n all_values v\n join\n set_values s on v.value_field = s.value_field\n\n)\n\nselect *\nfrom validation_errors\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "credit_card_amount", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "coupon_amount", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_coupon_amount"], "unique_id": "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_coupon_amount.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_coupon_amount", "alias": "not_null_orders_coupon_amount", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_coupon_amount.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.350801, "compiled_sql": "\n \n \n\n\n\nselect coupon_amount\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere coupon_amount is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "coupon_amount", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "bank_transfer_amount", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_bank_transfer_amount"], "unique_id": "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_bank_transfer_amount.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_bank_transfer_amount", "alias": "not_null_orders_bank_transfer_amount", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_bank_transfer_amount.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.351888, "compiled_sql": "\n \n \n\n\n\nselect bank_transfer_amount\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere bank_transfer_amount is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "bank_transfer_amount", "file_key_name": "models.orders"}, "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": {"raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "gift_card_amount", "model": "{{ get_where_subquery(ref('orders')) }}"}, "namespace": null}, "compiled": true, "resource_type": "test", "depends_on": {"macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], "nodes": ["model.jaffle_shop.orders"]}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "materialized": "test", "severity": "ERROR", "store_failures": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "database": "calm-pagoda-323403", "schema": "jaffle_shop_dbt_test__audit", "fqn": ["jaffle_shop", "not_null_orders_gift_card_amount"], "unique_id": "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "not_null_orders_gift_card_amount.sql", "original_file_path": "models/schema.yml", "name": "not_null_orders_gift_card_amount", "alias": "not_null_orders_gift_card_amount", "checksum": {"name": "none", "checksum": ""}, "tags": [], "refs": [["orders"]], "sources": [], "description": "", "columns": {}, "meta": {}, "docs": {"show": true}, "patch_path": null, "compiled_path": "target/compiled/jaffle_shop/models/schema.yml/not_null_orders_gift_card_amount.sql", "build_path": null, "deferred": false, "unrendered_config": {}, "created_at": 1655437984.353089, "compiled_sql": "\n \n \n\n\n\nselect gift_card_amount\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\nwhere gift_card_amount is null\n\n\n", "extra_ctes_injected": true, "extra_ctes": [], "relation_name": null, "column_name": "gift_card_amount", "file_key_name": "models.orders"}}, "sources": {}, "macros": {"macro.dbt_bigquery.date_sharded_table": {"unique_id": "macro.dbt_bigquery.date_sharded_table", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/etc.sql", "original_file_path": "macros/etc.sql", "name": "date_sharded_table", "macro_sql": "{% macro date_sharded_table(base_name) %}\n {{ return(base_name ~ \"[DBT__PARTITION_DATE]\") }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.822734}, "macro.dbt_bigquery.grant_access_to": {"unique_id": "macro.dbt_bigquery.grant_access_to", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/etc.sql", "original_file_path": "macros/etc.sql", "name": "grant_access_to", "macro_sql": "{% macro grant_access_to(entity, entity_type, role, grant_target_dict) -%}\n {% do adapter.grant_access_to(entity, entity_type, role, grant_target_dict) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8230212}, "macro.dbt_bigquery.get_partitions_metadata": {"unique_id": "macro.dbt_bigquery.get_partitions_metadata", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/etc.sql", "original_file_path": "macros/etc.sql", "name": "get_partitions_metadata", "macro_sql": "\n\n{%- macro get_partitions_metadata(table) -%}\n {%- if execute -%}\n {%- set res = adapter.get_partitions_metadata(table) -%}\n {{- return(res) -}}\n {%- endif -%}\n {{- return(None) -}}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8233569}, "macro.dbt_bigquery.bigquery__get_catalog": {"unique_id": "macro.dbt_bigquery.bigquery__get_catalog", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/catalog.sql", "original_file_path": "macros/catalog.sql", "name": "bigquery__get_catalog", "macro_sql": "{% macro bigquery__get_catalog(information_schema, schemas) -%}\n\n {%- if (schemas | length) == 0 -%}\n {# Hopefully nothing cares about the columns we return when there are no rows #}\n {%- set query = \"select 1 as id limit 0\" -%}\n {%- else -%}\n\n {%- set query -%}\n with tables as (\n select\n project_id as table_database,\n dataset_id as table_schema,\n table_id as original_table_name,\n\n concat(project_id, '.', dataset_id, '.', table_id) as relation_id,\n\n row_count,\n size_bytes as size_bytes,\n case\n when type = 1 then 'table'\n when type = 2 then 'view'\n else 'external'\n end as table_type,\n\n REGEXP_CONTAINS(table_id, '^.+[0-9]{8}$') and coalesce(type, 0) = 1 as is_date_shard,\n REGEXP_EXTRACT(table_id, '^(.+)[0-9]{8}$') as shard_base_name,\n REGEXP_EXTRACT(table_id, '^.+([0-9]{8})$') as shard_name\n\n from {{ information_schema.replace(information_schema_view='__TABLES__') }}\n where (\n {%- for schema in schemas -%}\n upper(dataset_id) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n ),\n\n extracted as (\n\n select *,\n case\n when is_date_shard then shard_base_name\n else original_table_name\n end as table_name\n\n from tables\n\n ),\n\n unsharded_tables as (\n\n select\n table_database,\n table_schema,\n table_name,\n coalesce(table_type, 'external') as table_type,\n is_date_shard,\n\n struct(\n min(shard_name) as shard_min,\n max(shard_name) as shard_max,\n count(*) as shard_count\n ) as table_shards,\n\n sum(size_bytes) as size_bytes,\n sum(row_count) as row_count,\n\n max(relation_id) as relation_id\n\n from extracted\n group by 1,2,3,4,5\n\n ),\n\n info_schema_columns as (\n\n select\n concat(table_catalog, '.', table_schema, '.', table_name) as relation_id,\n table_catalog as table_database,\n table_schema,\n table_name,\n\n -- use the \"real\" column name from the paths query below\n column_name as base_column_name,\n ordinal_position as column_index,\n\n is_partitioning_column,\n clustering_ordinal_position\n\n from {{ information_schema.replace(information_schema_view='COLUMNS') }}\n where ordinal_position is not null\n\n ),\n\n info_schema_column_paths as (\n\n select\n concat(table_catalog, '.', table_schema, '.', table_name) as relation_id,\n field_path as column_name,\n data_type as column_type,\n column_name as base_column_name,\n description as column_comment\n\n from {{ information_schema.replace(information_schema_view='COLUMN_FIELD_PATHS') }}\n\n ),\n\n columns as (\n\n select * except (base_column_name)\n from info_schema_columns\n join info_schema_column_paths using (relation_id, base_column_name)\n\n ),\n\n column_stats as (\n\n select\n table_database,\n table_schema,\n table_name,\n max(relation_id) as relation_id,\n max(case when is_partitioning_column = 'YES' then 1 else 0 end) = 1 as is_partitioned,\n max(case when is_partitioning_column = 'YES' then column_name else null end) as partition_column,\n max(case when clustering_ordinal_position is not null then 1 else 0 end) = 1 as is_clustered,\n array_to_string(\n array_agg(\n case\n when clustering_ordinal_position is not null then column_name\n else null\n end ignore nulls\n order by clustering_ordinal_position\n ), ', '\n ) as clustering_columns\n\n from columns\n group by 1,2,3\n\n )\n\n select\n unsharded_tables.table_database,\n unsharded_tables.table_schema,\n case\n when is_date_shard then concat(unsharded_tables.table_name, '*')\n else unsharded_tables.table_name\n end as table_name,\n unsharded_tables.table_type,\n\n -- coalesce name and type for External tables - these columns are not\n -- present in the COLUMN_FIELD_PATHS resultset\n coalesce(columns.column_name, '') as column_name,\n -- invent a row number to account for nested fields -- BQ does\n -- not treat these nested properties as independent fields\n row_number() over (\n partition by relation_id\n order by columns.column_index, columns.column_name\n ) as column_index,\n coalesce(columns.column_type, '') as column_type,\n columns.column_comment,\n\n 'Shard count' as `stats__date_shards__label`,\n table_shards.shard_count as `stats__date_shards__value`,\n 'The number of date shards in this table' as `stats__date_shards__description`,\n is_date_shard as `stats__date_shards__include`,\n\n 'Shard (min)' as `stats__date_shard_min__label`,\n table_shards.shard_min as `stats__date_shard_min__value`,\n 'The first date shard in this table' as `stats__date_shard_min__description`,\n is_date_shard as `stats__date_shard_min__include`,\n\n 'Shard (max)' as `stats__date_shard_max__label`,\n table_shards.shard_max as `stats__date_shard_max__value`,\n 'The last date shard in this table' as `stats__date_shard_max__description`,\n is_date_shard as `stats__date_shard_max__include`,\n\n '# Rows' as `stats__num_rows__label`,\n row_count as `stats__num_rows__value`,\n 'Approximate count of rows in this table' as `stats__num_rows__description`,\n (unsharded_tables.table_type = 'table') as `stats__num_rows__include`,\n\n 'Approximate Size' as `stats__num_bytes__label`,\n size_bytes as `stats__num_bytes__value`,\n 'Approximate size of table as reported by BigQuery' as `stats__num_bytes__description`,\n (unsharded_tables.table_type = 'table') as `stats__num_bytes__include`,\n\n 'Partitioned By' as `stats__partitioning_type__label`,\n partition_column as `stats__partitioning_type__value`,\n 'The partitioning column for this table' as `stats__partitioning_type__description`,\n is_partitioned as `stats__partitioning_type__include`,\n\n 'Clustered By' as `stats__clustering_fields__label`,\n clustering_columns as `stats__clustering_fields__value`,\n 'The clustering columns for this table' as `stats__clustering_fields__description`,\n is_clustered as `stats__clustering_fields__include`\n\n -- join using relation_id (an actual relation, not a shard prefix) to make\n -- sure that column metadata is picked up through the join. This will only\n -- return the column information for the \"max\" table in a date-sharded table set\n from unsharded_tables\n left join columns using (relation_id)\n left join column_stats using (relation_id)\n {%- endset -%}\n\n {%- endif -%}\n\n {{ return(run_query(query)) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.827381}, "macro.dbt_bigquery.partition_by": {"unique_id": "macro.dbt_bigquery.partition_by", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "partition_by", "macro_sql": "{% macro partition_by(partition_config) -%}\n {%- if partition_config is none -%}\n {% do return('') %}\n {%- elif partition_config.data_type | lower in ('date','timestamp','datetime') -%}\n partition by {{ partition_config.render() }}\n {%- elif partition_config.data_type | lower in ('int64') -%}\n {%- set range = partition_config.range -%}\n partition by range_bucket(\n {{ partition_config.field }},\n generate_array({{ range.start}}, {{ range.end }}, {{ range.interval }})\n )\n {%- endif -%}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.833942}, "macro.dbt_bigquery.cluster_by": {"unique_id": "macro.dbt_bigquery.cluster_by", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "cluster_by", "macro_sql": "{% macro cluster_by(raw_cluster_by) %}\n {%- if raw_cluster_by is not none -%}\n cluster by {% if raw_cluster_by is string -%}\n {% set raw_cluster_by = [raw_cluster_by] %}\n {%- endif -%}\n {%- for cluster in raw_cluster_by -%}\n {{ cluster }}\n {%- if not loop.last -%}, {% endif -%}\n {%- endfor -%}\n\n {% endif %}\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8343801}, "macro.dbt_bigquery.bigquery_options": {"unique_id": "macro.dbt_bigquery.bigquery_options", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery_options", "macro_sql": "{% macro bigquery_options(opts) %}\n {% set options -%}\n OPTIONS({% for opt_key, opt_val in opts.items() %}\n {{ opt_key }}={{ opt_val }}{{ \",\" if not loop.last }}\n {% endfor %})\n {%- endset %}\n {%- do return(options) -%}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8347871}, "macro.dbt_bigquery.bigquery_table_options": {"unique_id": "macro.dbt_bigquery.bigquery_table_options", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery_table_options", "macro_sql": "{% macro bigquery_table_options(config, node, temporary) %}\n {% set opts = adapter.get_table_options(config, node, temporary) %}\n {%- do return(bigquery_options(opts)) -%}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery_options"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.835086}, "macro.dbt_bigquery.bigquery__create_table_as": {"unique_id": "macro.dbt_bigquery.bigquery__create_table_as", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__create_table_as", "macro_sql": "{% macro bigquery__create_table_as(temporary, relation, sql) -%}\n {%- set raw_partition_by = config.get('partition_by', none) -%}\n {%- set raw_cluster_by = config.get('cluster_by', none) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create or replace table {{ relation }}\n {{ partition_by(partition_config) }}\n {{ cluster_by(raw_cluster_by) }}\n {{ bigquery_table_options(config, model, temporary) }}\n as (\n {{ sql }}\n );\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.partition_by", "macro.dbt_bigquery.cluster_by", "macro.dbt_bigquery.bigquery_table_options"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.835826}, "macro.dbt_bigquery.bigquery_view_options": {"unique_id": "macro.dbt_bigquery.bigquery_view_options", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery_view_options", "macro_sql": "{% macro bigquery_view_options(config, node) %}\n {% set opts = adapter.get_view_options(config, node) %}\n {%- do return(bigquery_options(opts)) -%}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery_options"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.836097}, "macro.dbt_bigquery.bigquery__create_view_as": {"unique_id": "macro.dbt_bigquery.bigquery__create_view_as", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__create_view_as", "macro_sql": "{% macro bigquery__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create or replace view {{ relation }}\n {{ bigquery_view_options(config, model) }}\n as {{ sql }};\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery_view_options"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.836461}, "macro.dbt_bigquery.bigquery__create_schema": {"unique_id": "macro.dbt_bigquery.bigquery__create_schema", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__create_schema", "macro_sql": "{% macro bigquery__create_schema(relation) -%}\n {{ adapter.create_schema(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.836601}, "macro.dbt_bigquery.bigquery__drop_schema": {"unique_id": "macro.dbt_bigquery.bigquery__drop_schema", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__drop_schema", "macro_sql": "{% macro bigquery__drop_schema(relation) -%}\n {{ adapter.drop_schema(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8367429}, "macro.dbt_bigquery.bigquery__drop_relation": {"unique_id": "macro.dbt_bigquery.bigquery__drop_relation", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__drop_relation", "macro_sql": "{% macro bigquery__drop_relation(relation) -%}\n {% call statement('drop_relation') -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8369558}, "macro.dbt_bigquery.bigquery__get_columns_in_relation": {"unique_id": "macro.dbt_bigquery.bigquery__get_columns_in_relation", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__get_columns_in_relation", "macro_sql": "{% macro bigquery__get_columns_in_relation(relation) -%}\n {{ return(adapter.get_columns_in_relation(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.837121}, "macro.dbt_bigquery.bigquery__list_relations_without_caching": {"unique_id": "macro.dbt_bigquery.bigquery__list_relations_without_caching", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__list_relations_without_caching", "macro_sql": "{% macro bigquery__list_relations_without_caching(schema_relation) -%}\n {{ return(adapter.list_relations_without_caching(schema_relation)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.837281}, "macro.dbt_bigquery.bigquery__current_timestamp": {"unique_id": "macro.dbt_bigquery.bigquery__current_timestamp", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__current_timestamp", "macro_sql": "{% macro bigquery__current_timestamp() -%}\n CURRENT_TIMESTAMP()\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8373609}, "macro.dbt_bigquery.bigquery__snapshot_string_as_time": {"unique_id": "macro.dbt_bigquery.bigquery__snapshot_string_as_time", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__snapshot_string_as_time", "macro_sql": "{% macro bigquery__snapshot_string_as_time(timestamp) -%}\n {%- set result = 'TIMESTAMP(\"' ~ timestamp ~ '\")' -%}\n {{ return(result) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.837557}, "macro.dbt_bigquery.bigquery__list_schemas": {"unique_id": "macro.dbt_bigquery.bigquery__list_schemas", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__list_schemas", "macro_sql": "{% macro bigquery__list_schemas(database) -%}\n {{ return(adapter.list_schemas(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.837728}, "macro.dbt_bigquery.bigquery__check_schema_exists": {"unique_id": "macro.dbt_bigquery.bigquery__check_schema_exists", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__check_schema_exists", "macro_sql": "{% macro bigquery__check_schema_exists(information_schema, schema) %}\n {{ return(adapter.check_schema_exists(information_schema.database, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8379269}, "macro.dbt_bigquery.bigquery__persist_docs": {"unique_id": "macro.dbt_bigquery.bigquery__persist_docs", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__persist_docs", "macro_sql": "{% macro bigquery__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.838242}, "macro.dbt_bigquery.bigquery__alter_column_comment": {"unique_id": "macro.dbt_bigquery.bigquery__alter_column_comment", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__alter_column_comment", "macro_sql": "{% macro bigquery__alter_column_comment(relation, column_dict) -%}\n {% do adapter.update_columns(relation, column_dict) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8384151}, "macro.dbt_bigquery.bigquery__rename_relation": {"unique_id": "macro.dbt_bigquery.bigquery__rename_relation", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__rename_relation", "macro_sql": "{% macro bigquery__rename_relation(from_relation, to_relation) -%}\n {% do adapter.rename_relation(from_relation, to_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.838593}, "macro.dbt_bigquery.bigquery__alter_relation_add_columns": {"unique_id": "macro.dbt_bigquery.bigquery__alter_relation_add_columns", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__alter_relation_add_columns", "macro_sql": "{% macro bigquery__alter_relation_add_columns(relation, add_columns) %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {{ return(run_query(sql)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8391092}, "macro.dbt_bigquery.bigquery__alter_relation_drop_columns": {"unique_id": "macro.dbt_bigquery.bigquery__alter_relation_drop_columns", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__alter_relation_drop_columns", "macro_sql": "{% macro bigquery__alter_relation_drop_columns(relation, drop_columns) %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in drop_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {{ return(run_query(sql)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.839526}, "macro.dbt_bigquery.bigquery__alter_column_type": {"unique_id": "macro.dbt_bigquery.bigquery__alter_column_type", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__alter_column_type", "macro_sql": "{% macro bigquery__alter_column_type(relation, column_name, new_column_type) -%}\n {#-- Changing a column's data type using a query requires you to scan the entire table.\n The query charges can be significant if the table is very large.\n\n https://cloud.google.com/bigquery/docs/manually-changing-schemas#changing_a_columns_data_type\n #}\n {% set relation_columns = get_columns_in_relation(relation) %}\n\n {% set sql %}\n select\n {%- for col in relation_columns -%}\n {% if col.column == column_name %}\n CAST({{ col.quoted }} AS {{ new_column_type }}) AS {{ col.quoted }}\n {%- else %}\n {{ col.quoted }}\n {%- endif %}\n {%- if not loop.last %},{% endif -%}\n {%- endfor %}\n from {{ relation }}\n {% endset %}\n\n {% call statement('alter_column_type') %}\n {{ create_table_as(False, relation, sql)}}\n {%- endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_relation", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8402822}, "macro.dbt_bigquery.bigquery__test_unique": {"unique_id": "macro.dbt_bigquery.bigquery__test_unique", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__test_unique", "macro_sql": "{% macro bigquery__test_unique(model, column_name) %}\n\nwith dbt_test__target as (\n\n select {{ column_name }} as unique_field\n from {{ model }}\n where {{ column_name }} is not null\n\n)\n\nselect\n unique_field,\n count(*) as n_records\n\nfrom dbt_test__target\ngroup by unique_field\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.840468}, "macro.dbt_bigquery.bigquery__upload_file": {"unique_id": "macro.dbt_bigquery.bigquery__upload_file", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "name": "bigquery__upload_file", "macro_sql": "{% macro bigquery__upload_file(local_file_path, database, table_schema, table_name) %}\n\n {{ log(\"kwargs: \" ~ kwargs) }}\n\n {% do adapter.upload_file(local_file_path, database, table_schema, table_name, kwargs=kwargs) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8407822}, "macro.dbt_bigquery.bigquery__create_csv_table": {"unique_id": "macro.dbt_bigquery.bigquery__create_csv_table", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "name": "bigquery__create_csv_table", "macro_sql": "{% macro bigquery__create_csv_table(model, agate_table) %}\n -- no-op\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8412101}, "macro.dbt_bigquery.bigquery__reset_csv_table": {"unique_id": "macro.dbt_bigquery.bigquery__reset_csv_table", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "name": "bigquery__reset_csv_table", "macro_sql": "{% macro bigquery__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.841385}, "macro.dbt_bigquery.bigquery__load_csv_rows": {"unique_id": "macro.dbt_bigquery.bigquery__load_csv_rows", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "name": "bigquery__load_csv_rows", "macro_sql": "{% macro bigquery__load_csv_rows(model, agate_table) %}\n\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {{ adapter.load_dataframe(model['database'], model['schema'], model['alias'],\n \t\t\t\t\t\t\tagate_table, column_override) }}\n {% if config.persist_relation_docs() and 'description' in model %}\n\n \t{{ adapter.update_table_description(model['database'], model['schema'], model['alias'], model['description']) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.842079}, "macro.dbt_bigquery.bigquery__handle_existing_table": {"unique_id": "macro.dbt_bigquery.bigquery__handle_existing_table", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "name": "bigquery__handle_existing_table", "macro_sql": "{% macro bigquery__handle_existing_table(full_refresh, old_relation) %}\n {%- if full_refresh -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- else -%}\n {{ exceptions.relation_wrong_type(old_relation, 'view') }}\n {%- endif -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8427289}, "macro.dbt_bigquery.materialization_view_bigquery": {"unique_id": "macro.dbt_bigquery.materialization_view_bigquery", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "name": "materialization_view_bigquery", "macro_sql": "{% materialization view, adapter='bigquery' -%}\n {% set to_return = create_or_replace_view() %}\n\n {% set target_relation = this.incorporate(type='view') %}\n {% do persist_docs(target_relation, model) %}\n\n {% if config.get('grant_access_to') %}\n {% for grant_target_dict in config.get('grant_access_to') %}\n {% do adapter.grant_access_to(this, 'view', None, grant_target_dict) %}\n {% endfor %}\n {% endif %}\n\n {% do return(to_return) %}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_or_replace_view", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.843381}, "macro.dbt_bigquery.materialization_table_bigquery": {"unique_id": "macro.dbt_bigquery.materialization_table_bigquery", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "name": "materialization_table_bigquery", "macro_sql": "{% materialization table, adapter='bigquery' -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}\n {%- set target_relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n {#\n We only need to drop this thing if it is not a table.\n If it _is_ already a table, then we can overwrite it without downtime\n Unlike table -> view, no need for `--full-refresh`: dropping a view is no big deal\n #}\n {%- if exists_not_as_table -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif -%}\n\n -- build model\n {%- set raw_partition_by = config.get('partition_by', none) -%}\n {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%}\n {%- set cluster_by = config.get('cluster_by', none) -%}\n {% if not adapter.is_replaceable(old_relation, partition_by, cluster_by) %}\n {% do log(\"Hard refreshing \" ~ old_relation ~ \" because it is not replaceable\") %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n {% call statement('main') -%}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall -%}\n\n {{ run_hooks(post_hooks) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.845387}, "macro.dbt_bigquery.materialization_copy_bigquery": {"unique_id": "macro.dbt_bigquery.materialization_copy_bigquery", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/copy.sql", "original_file_path": "macros/materializations/copy.sql", "name": "materialization_copy_bigquery", "macro_sql": "{% materialization copy, adapter='bigquery' -%}\n\n {# Setup #}\n {{ run_hooks(pre_hooks) }}\n\n {% set destination = this.incorporate(type='table') %}\n\n {# there can be several ref() or source() according to BQ copy API docs #}\n {# cycle over ref() and source() to create source tables array #}\n {% set source_array = [] %}\n {% for ref_table in model.refs %}\n {{ source_array.append(ref(*ref_table)) }}\n {% endfor %}\n\n {% for src_table in model.sources %}\n {{ source_array.append(source(*src_table)) }}\n {% endfor %}\n\n {# Call adapter's copy_table function #}\n {%- set result_str = adapter.copy_table(\n source_array,\n destination,\n config.get('copy_materialization', default = 'table')) -%}\n\n {{ store_result('main', response=result_str) }}\n\n {# Clean up #}\n {{ run_hooks(post_hooks) }}\n {{ adapter.commit() }}\n\n {{ return({'relations': [destination]}) }}\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8467178}, "macro.dbt_bigquery.declare_dbt_max_partition": {"unique_id": "macro.dbt_bigquery.declare_dbt_max_partition", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/incremental.sql", "original_file_path": "macros/materializations/incremental.sql", "name": "declare_dbt_max_partition", "macro_sql": "{% macro declare_dbt_max_partition(relation, partition_by, sql) %}\n\n {% if '_dbt_max_partition' in sql %}\n\n declare _dbt_max_partition {{ partition_by.data_type }} default (\n select max({{ partition_by.field }}) from {{ this }}\n where {{ partition_by.field }} is not null\n );\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.848785}, "macro.dbt_bigquery.dbt_bigquery_validate_get_incremental_strategy": {"unique_id": "macro.dbt_bigquery.dbt_bigquery_validate_get_incremental_strategy", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/incremental.sql", "original_file_path": "macros/materializations/incremental.sql", "name": "dbt_bigquery_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_bigquery_validate_get_incremental_strategy(config) %}\n {#-- Find and validate the incremental strategy #}\n {%- set strategy = config.get(\"incremental_strategy\", default=\"merge\") -%}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ strategy }}\n Expected one of: 'merge', 'insert_overwrite'\n {%- endset %}\n {% if strategy not in ['merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {% endif %}\n\n {% do return(strategy) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.849278}, "macro.dbt_bigquery.bq_insert_overwrite": {"unique_id": "macro.dbt_bigquery.bq_insert_overwrite", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/incremental.sql", "original_file_path": "macros/materializations/incremental.sql", "name": "bq_insert_overwrite", "macro_sql": "{% macro bq_insert_overwrite(\n tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists\n) %}\n\n {% if partitions is not none and partitions != [] %} {# static #}\n\n {% set predicate -%}\n {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in (\n {{ partitions | join (', ') }}\n )\n {%- endset %}\n\n {%- set source_sql -%}\n (\n {{sql}}\n )\n {%- endset -%}\n\n {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=true) }}\n\n {% else %} {# dynamic #}\n\n {% set predicate -%}\n {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in unnest(dbt_partitions_for_replacement)\n {%- endset %}\n\n {%- set source_sql -%}\n (\n select * from {{ tmp_relation }}\n )\n {%- endset -%}\n\n -- generated script to merge partitions into {{ target_relation }}\n declare dbt_partitions_for_replacement array<{{ partition_by.data_type }}>;\n\n {# have we already created the temp table to check for schema changes? #}\n {% if not tmp_relation_exists %}\n {{ declare_dbt_max_partition(this, partition_by, sql) }}\n\n -- 1. create a temp table\n {{ create_table_as(True, tmp_relation, sql) }}\n {% else %}\n -- 1. temp table already exists, we used it to check for schema changes\n {% endif %}\n\n -- 2. define partitions to update\n set (dbt_partitions_for_replacement) = (\n select as struct\n array_agg(distinct {{ partition_by.render() }})\n from {{ tmp_relation }}\n );\n\n {#\n TODO: include_sql_header is a hack; consider a better approach that includes\n the sql_header at the materialization-level instead\n #}\n -- 3. run the merge statement\n {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=false) }};\n\n -- 4. clean up the temp table\n drop table if exists {{ tmp_relation }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql", "macro.dbt_bigquery.declare_dbt_max_partition", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8506649}, "macro.dbt_bigquery.bq_generate_incremental_build_sql": {"unique_id": "macro.dbt_bigquery.bq_generate_incremental_build_sql", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/incremental.sql", "original_file_path": "macros/materializations/incremental.sql", "name": "bq_generate_incremental_build_sql", "macro_sql": "{% macro bq_generate_incremental_build_sql(\n strategy, tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists\n) %}\n {#-- if partitioned, use BQ scripting to get the range of partition values to be updated --#}\n {% if strategy == 'insert_overwrite' %}\n\n {% set missing_partition_msg -%}\n The 'insert_overwrite' strategy requires the `partition_by` config.\n {%- endset %}\n {% if partition_by is none %}\n {% do exceptions.raise_compiler_error(missing_partition_msg) %}\n {% endif %}\n\n {% set build_sql = bq_insert_overwrite(\n tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, on_schema_change\n ) %}\n\n {% else %} {# strategy == 'merge' #}\n {%- set source_sql -%}\n {%- if tmp_relation_exists -%}\n (\n select * from {{ tmp_relation }}\n )\n {%- else -%} {#-- wrap sql in parens to make it a subquery --#}\n (\n {{sql}}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set build_sql = get_merge_sql(target_relation, source_sql, unique_key, dest_columns) %}\n\n {% endif %}\n\n {{ return(build_sql) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bq_insert_overwrite", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.851686}, "macro.dbt_bigquery.materialization_incremental_bigquery": {"unique_id": "macro.dbt_bigquery.materialization_incremental_bigquery", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/incremental.sql", "original_file_path": "macros/materializations/incremental.sql", "name": "materialization_incremental_bigquery", "macro_sql": "{% materialization incremental, adapter='bigquery' -%}\n\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set target_relation = this %}\n {%- set existing_relation = load_relation(this) %}\n {%- set tmp_relation = make_temp_relation(this) %}\n\n {#-- Validate early so we don't run SQL if the strategy is invalid --#}\n {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%}\n\n {%- set raw_partition_by = config.get('partition_by', none) -%}\n {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%}\n {%- set partitions = config.get('partitions', none) -%}\n {%- set cluster_by = config.get('cluster_by', none) -%}\n\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n\n {{ run_hooks(pre_hooks) }}\n\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n\n {% elif existing_relation.is_view %}\n {#-- There's no way to atomically replace a view with a table on BQ --#}\n {{ adapter.drop_relation(existing_relation) }}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n\n {% elif full_refresh_mode %}\n {#-- If the partition/cluster config has changed, then we must drop and recreate --#}\n {% if not adapter.is_replaceable(existing_relation, partition_by, cluster_by) %}\n {% do log(\"Hard refreshing \" ~ existing_relation ~ \" because it is not replaceable\") %}\n {{ adapter.drop_relation(existing_relation) }}\n {% endif %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n\n {% else %}\n {% set tmp_relation_exists = false %}\n {% if on_schema_change != 'ignore' %} {# Check first, since otherwise we may not build a temp table #}\n {% do run_query(\n declare_dbt_max_partition(this, partition_by, sql) + create_table_as(True, tmp_relation, sql)\n ) %}\n {% set tmp_relation_exists = true %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}\n {% endif %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n {% set build_sql = bq_generate_incremental_build_sql(\n strategy, tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists\n ) %}\n\n {% endif %}\n\n {%- call statement('main') -%}\n {{ build_sql }}\n {% endcall %}\n\n {{ run_hooks(post_hooks) }}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt_bigquery.dbt_bigquery_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.run_query", "macro.dbt_bigquery.declare_dbt_max_partition", "macro.dbt.process_schema_changes", "macro.dbt_bigquery.bq_generate_incremental_build_sql", "macro.dbt.statement", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.85453}, "macro.dbt_bigquery.bigquery__snapshot_hash_arguments": {"unique_id": "macro.dbt_bigquery.bigquery__snapshot_hash_arguments", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "name": "bigquery__snapshot_hash_arguments", "macro_sql": "{% macro bigquery__snapshot_hash_arguments(args) -%}\n to_hex(md5(concat({%- for arg in args -%}\n coalesce(cast({{ arg }} as string), ''){% if not loop.last %}, '|',{% endif -%}\n {%- endfor -%}\n )))\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.855057}, "macro.dbt_bigquery.bigquery__create_columns": {"unique_id": "macro.dbt_bigquery.bigquery__create_columns", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "name": "bigquery__create_columns", "macro_sql": "{% macro bigquery__create_columns(relation, columns) %}\n {{ adapter.alter_table_add_columns(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.855243}, "macro.dbt_bigquery.bigquery__post_snapshot": {"unique_id": "macro.dbt_bigquery.bigquery__post_snapshot", "package_name": "dbt_bigquery", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/bigquery", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "name": "bigquery__post_snapshot", "macro_sql": "{% macro bigquery__post_snapshot(staging_relation) %}\n -- Clean up the snapshot temp table\n {% do drop_relation(staging_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.855387}, "macro.dbt.run_hooks": {"unique_id": "macro.dbt.run_hooks", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.856645}, "macro.dbt.make_hook_config": {"unique_id": "macro.dbt.make_hook_config", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.856859}, "macro.dbt.before_begin": {"unique_id": "macro.dbt.before_begin", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.85701}, "macro.dbt.in_transaction": {"unique_id": "macro.dbt.in_transaction", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.857162}, "macro.dbt.after_commit": {"unique_id": "macro.dbt.after_commit", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "name": "after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8573382}, "macro.dbt.set_sql_header": {"unique_id": "macro.dbt.set_sql_header", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.857846}, "macro.dbt.should_full_refresh": {"unique_id": "macro.dbt.should_full_refresh", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.858189}, "macro.dbt.should_store_failures": {"unique_id": "macro.dbt.should_store_failures", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "name": "should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8585181}, "macro.dbt.snapshot_merge_sql": {"unique_id": "macro.dbt.snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.859058}, "macro.dbt.default__snapshot_merge_sql": {"unique_id": "macro.dbt.default__snapshot_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "name": "default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.859348}, "macro.dbt.strategy_dispatch": {"unique_id": "macro.dbt.strategy_dispatch", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.86339}, "macro.dbt.snapshot_hash_arguments": {"unique_id": "macro.dbt.snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.863574}, "macro.dbt.default__snapshot_hash_arguments": {"unique_id": "macro.dbt.default__snapshot_hash_arguments", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.863817}, "macro.dbt.snapshot_get_time": {"unique_id": "macro.dbt.snapshot_get_time", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_get_time", "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.863969}, "macro.dbt.default__snapshot_get_time": {"unique_id": "macro.dbt.default__snapshot_get_time", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.864068}, "macro.dbt.snapshot_timestamp_strategy": {"unique_id": "macro.dbt.snapshot_timestamp_strategy", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.864872}, "macro.dbt.snapshot_string_as_time": {"unique_id": "macro.dbt.snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.865063}, "macro.dbt.default__snapshot_string_as_time": {"unique_id": "macro.dbt.default__snapshot_string_as_time", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.865248}, "macro.dbt.snapshot_check_all_get_existing_columns": {"unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8663309}, "macro.dbt.snapshot_check_strategy": {"unique_id": "macro.dbt.snapshot_check_strategy", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "name": "snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.868126}, "macro.dbt.create_columns": {"unique_id": "macro.dbt.create_columns", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.87232}, "macro.dbt.default__create_columns": {"unique_id": "macro.dbt.default__create_columns", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8726301}, "macro.dbt.post_snapshot": {"unique_id": "macro.dbt.post_snapshot", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.872812}, "macro.dbt.default__post_snapshot": {"unique_id": "macro.dbt.default__post_snapshot", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.872902}, "macro.dbt.get_true_sql": {"unique_id": "macro.dbt.get_true_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.873055}, "macro.dbt.default__get_true_sql": {"unique_id": "macro.dbt.default__get_true_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8731751}, "macro.dbt.snapshot_staging_table": {"unique_id": "macro.dbt.snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.873394}, "macro.dbt.default__snapshot_staging_table": {"unique_id": "macro.dbt.default__snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8743622}, "macro.dbt.build_snapshot_table": {"unique_id": "macro.dbt.build_snapshot_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8745651}, "macro.dbt.default__build_snapshot_table": {"unique_id": "macro.dbt.default__build_snapshot_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.874839}, "macro.dbt.build_snapshot_staging_table": {"unique_id": "macro.dbt.build_snapshot_staging_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "name": "build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8753128}, "macro.dbt.materialization_snapshot_default": {"unique_id": "macro.dbt.materialization_snapshot_default", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "name": "materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.881349}, "macro.dbt.materialization_test_default": {"unique_id": "macro.dbt.materialization_test_default", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "name": "materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.883666}, "macro.dbt.get_test_sql": {"unique_id": "macro.dbt.get_test_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.884223}, "macro.dbt.default__get_test_sql": {"unique_id": "macro.dbt.default__get_test_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "name": "default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.884564}, "macro.dbt.get_where_subquery": {"unique_id": "macro.dbt.get_where_subquery", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.8850482}, "macro.dbt.default__get_where_subquery": {"unique_id": "macro.dbt.default__get_where_subquery", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "name": "default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.885463}, "macro.dbt.get_quoted_csv": {"unique_id": "macro.dbt.get_quoted_csv", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.886404}, "macro.dbt.diff_columns": {"unique_id": "macro.dbt.diff_columns", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.887006}, "macro.dbt.diff_column_data_types": {"unique_id": "macro.dbt.diff_column_data_types", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "name": "diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.887685}, "macro.dbt.get_merge_sql": {"unique_id": "macro.dbt.get_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.893273}, "macro.dbt.default__get_merge_sql": {"unique_id": "macro.dbt.default__get_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute=\"quoted\") | list) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.894926}, "macro.dbt.get_delete_insert_merge_sql": {"unique_id": "macro.dbt.get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.895246}, "macro.dbt.default__get_delete_insert_merge_sql": {"unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last }}\n {% endfor %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.896078}, "macro.dbt.get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.896371}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "name": "default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.897063}, "macro.dbt.is_incremental": {"unique_id": "macro.dbt.is_incremental", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "name": "is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.897841}, "macro.dbt.materialization_incremental_default": {"unique_id": "macro.dbt.materialization_incremental_default", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "name": "materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + \"__dbt_backup\" %}\n\n -- the intermediate_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {% set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier,\n schema=schema,\n database=database) %}\n {% set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {# -- first check whether we want to full refresh for source view or config reasons #}\n {% set trigger_full_refresh = (full_refresh_mode or existing_relation.is_view) %}\n\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n{% elif trigger_full_refresh %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set tmp_identifier = model['name'] + '__dbt_tmp' %}\n {% set backup_identifier = model['name'] + '__dbt_backup' %}\n {% set intermediate_relation = existing_relation.incorporate(path={\"identifier\": tmp_identifier}) %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n\n {% set build_sql = create_table_as(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n {% set build_sql = get_delete_insert_merge_sql(target_relation, tmp_relation, unique_key, dest_columns) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% endif %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.get_delete_insert_merge_sql", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.903169}, "macro.dbt.incremental_validate_on_schema_change": {"unique_id": "macro.dbt.incremental_validate_on_schema_change", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.908252}, "macro.dbt.check_for_schema_changes": {"unique_id": "macro.dbt.check_for_schema_changes", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9096289}, "macro.dbt.sync_column_schemas": {"unique_id": "macro.dbt.sync_column_schemas", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.910963}, "macro.dbt.process_schema_changes": {"unique_id": "macro.dbt.process_schema_changes", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "name": "process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9117582}, "macro.dbt.materialization_table_default": {"unique_id": "macro.dbt.materialization_table_default", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/table.sql", "original_file_path": "macros/materializations/models/table/table.sql", "name": "materialization_table_default", "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier,\n schema=schema,\n database=database) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9159338}, "macro.dbt.get_create_table_as_sql": {"unique_id": "macro.dbt.get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.916485}, "macro.dbt.default__get_create_table_as_sql": {"unique_id": "macro.dbt.default__get_create_table_as_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9166842}, "macro.dbt.create_table_as": {"unique_id": "macro.dbt.create_table_as", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.916905}, "macro.dbt.default__create_table_as": {"unique_id": "macro.dbt.default__create_table_as", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/table/create_table_as.sql", "original_file_path": "macros/materializations/models/table/create_table_as.sql", "name": "default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.917351}, "macro.dbt.materialization_view_default": {"unique_id": "macro.dbt.materialization_view_default", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/view.sql", "original_file_path": "macros/materializations/models/view/view.sql", "name": "materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier,\n schema=schema,\n database=database) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,\n schema=schema,\n database=database) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(old_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_view_as", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.921458}, "macro.dbt.handle_existing_table": {"unique_id": "macro.dbt.handle_existing_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.921865}, "macro.dbt.default__handle_existing_table": {"unique_id": "macro.dbt.default__handle_existing_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/helpers.sql", "original_file_path": "macros/materializations/models/view/helpers.sql", "name": "default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.922109}, "macro.dbt.create_or_replace_view": {"unique_id": "macro.dbt.create_or_replace_view", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_or_replace_view.sql", "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", "name": "create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9235158}, "macro.dbt.get_create_view_as_sql": {"unique_id": "macro.dbt.get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.923979}, "macro.dbt.default__get_create_view_as_sql": {"unique_id": "macro.dbt.default__get_create_view_as_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.924156}, "macro.dbt.create_view_as": {"unique_id": "macro.dbt.create_view_as", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.92435}, "macro.dbt.default__create_view_as": {"unique_id": "macro.dbt.default__create_view_as", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/models/view/create_view_as.sql", "original_file_path": "macros/materializations/models/view/create_view_as.sql", "name": "default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9246342}, "macro.dbt.materialization_seed_default": {"unique_id": "macro.dbt.materialization_seed_default", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "name": "materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9278111}, "macro.dbt.create_csv_table": {"unique_id": "macro.dbt.create_csv_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9323468}, "macro.dbt.default__create_csv_table": {"unique_id": "macro.dbt.default__create_csv_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9333389}, "macro.dbt.reset_csv_table": {"unique_id": "macro.dbt.reset_csv_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.933593}, "macro.dbt.default__reset_csv_table": {"unique_id": "macro.dbt.default__reset_csv_table", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.934122}, "macro.dbt.get_binding_char": {"unique_id": "macro.dbt.get_binding_char", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.934277}, "macro.dbt.default__get_binding_char": {"unique_id": "macro.dbt.default__get_binding_char", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.934398}, "macro.dbt.get_batch_size": {"unique_id": "macro.dbt.get_batch_size", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.934568}, "macro.dbt.default__get_batch_size": {"unique_id": "macro.dbt.default__get_batch_size", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.93469}, "macro.dbt.get_seed_column_quoted_csv": {"unique_id": "macro.dbt.get_seed_column_quoted_csv", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.935207}, "macro.dbt.load_csv_rows": {"unique_id": "macro.dbt.load_csv_rows", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9354649}, "macro.dbt.default__load_csv_rows": {"unique_id": "macro.dbt.default__load_csv_rows", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "name": "default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.936804}, "macro.dbt.generate_alias_name": {"unique_id": "macro.dbt.generate_alias_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.937299}, "macro.dbt.default__generate_alias_name": {"unique_id": "macro.dbt.default__generate_alias_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "name": "default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.937549}, "macro.dbt.generate_schema_name": {"unique_id": "macro.dbt.generate_schema_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9381819}, "macro.dbt.default__generate_schema_name": {"unique_id": "macro.dbt.default__generate_schema_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.938464}, "macro.dbt.generate_schema_name_for_env": {"unique_id": "macro.dbt.generate_schema_name_for_env", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "name": "generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9388092}, "macro.dbt.generate_database_name": {"unique_id": "macro.dbt.generate_database_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.939322}, "macro.dbt.default__generate_database_name": {"unique_id": "macro.dbt.default__generate_database_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "name": "default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.93959}, "macro.dbt.default__test_relationships": {"unique_id": "macro.dbt.default__test_relationships", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "name": "default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.940039}, "macro.dbt.default__test_not_null": {"unique_id": "macro.dbt.default__test_not_null", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "name": "default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9404402}, "macro.dbt.default__test_unique": {"unique_id": "macro.dbt.default__test_unique", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "name": "default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.940788}, "macro.dbt.default__test_accepted_values": {"unique_id": "macro.dbt.default__test_accepted_values", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "name": "default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9414752}, "macro.dbt.statement": {"unique_id": "macro.dbt.statement", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "statement", "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9427469}, "macro.dbt.noop_statement": {"unique_id": "macro.dbt.noop_statement", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9433851}, "macro.dbt.run_query": {"unique_id": "macro.dbt.run_query", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "name": "run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.943706}, "macro.dbt.convert_datetime": {"unique_id": "macro.dbt.convert_datetime", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.945686}, "macro.dbt.dates_in_range": {"unique_id": "macro.dbt.dates_in_range", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.947078}, "macro.dbt.partition_range": {"unique_id": "macro.dbt.partition_range", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.947909}, "macro.dbt.py_current_timestring": {"unique_id": "macro.dbt.py_current_timestring", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "name": "py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.948159}, "macro.dbt.create_schema": {"unique_id": "macro.dbt.create_schema", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.948652}, "macro.dbt.default__create_schema": {"unique_id": "macro.dbt.default__create_schema", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.948855}, "macro.dbt.drop_schema": {"unique_id": "macro.dbt.drop_schema", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.949025}, "macro.dbt.default__drop_schema": {"unique_id": "macro.dbt.default__drop_schema", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "name": "default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9492202}, "macro.dbt.get_create_index_sql": {"unique_id": "macro.dbt.get_create_index_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.949792}, "macro.dbt.default__get_create_index_sql": {"unique_id": "macro.dbt.default__get_create_index_sql", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9499378}, "macro.dbt.create_indexes": {"unique_id": "macro.dbt.create_indexes", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9501061}, "macro.dbt.default__create_indexes": {"unique_id": "macro.dbt.default__create_indexes", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "name": "default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9505439}, "macro.dbt.make_temp_relation": {"unique_id": "macro.dbt.make_temp_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.952354}, "macro.dbt.default__make_temp_relation": {"unique_id": "macro.dbt.default__make_temp_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.952684}, "macro.dbt.drop_relation": {"unique_id": "macro.dbt.drop_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.952884}, "macro.dbt.default__drop_relation": {"unique_id": "macro.dbt.default__drop_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9531121}, "macro.dbt.truncate_relation": {"unique_id": "macro.dbt.truncate_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.953307}, "macro.dbt.default__truncate_relation": {"unique_id": "macro.dbt.default__truncate_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9534762}, "macro.dbt.rename_relation": {"unique_id": "macro.dbt.rename_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.953695}, "macro.dbt.default__rename_relation": {"unique_id": "macro.dbt.default__rename_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.953998}, "macro.dbt.get_or_create_relation": {"unique_id": "macro.dbt.get_or_create_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.954276}, "macro.dbt.default__get_or_create_relation": {"unique_id": "macro.dbt.default__get_or_create_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9549508}, "macro.dbt.load_relation": {"unique_id": "macro.dbt.load_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.955223}, "macro.dbt.drop_relation_if_exists": {"unique_id": "macro.dbt.drop_relation_if_exists", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "name": "drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.95545}, "macro.dbt.current_timestamp": {"unique_id": "macro.dbt.current_timestamp", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9560409}, "macro.dbt.default__current_timestamp": {"unique_id": "macro.dbt.default__current_timestamp", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.956197}, "macro.dbt.collect_freshness": {"unique_id": "macro.dbt.collect_freshness", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.956445}, "macro.dbt.default__collect_freshness": {"unique_id": "macro.dbt.default__collect_freshness", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "name": "default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_utils.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.956907}, "macro.dbt.alter_column_comment": {"unique_id": "macro.dbt.alter_column_comment", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9577}, "macro.dbt.default__alter_column_comment": {"unique_id": "macro.dbt.default__alter_column_comment", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.957877}, "macro.dbt.alter_relation_comment": {"unique_id": "macro.dbt.alter_relation_comment", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9580958}, "macro.dbt.default__alter_relation_comment": {"unique_id": "macro.dbt.default__alter_relation_comment", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.958275}, "macro.dbt.persist_docs": {"unique_id": "macro.dbt.persist_docs", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.958569}, "macro.dbt.default__persist_docs": {"unique_id": "macro.dbt.default__persist_docs", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "name": "default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9591029}, "macro.dbt.get_catalog": {"unique_id": "macro.dbt.get_catalog", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.960725}, "macro.dbt.default__get_catalog": {"unique_id": "macro.dbt.default__get_catalog", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.961003}, "macro.dbt.information_schema_name": {"unique_id": "macro.dbt.information_schema_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9612029}, "macro.dbt.default__information_schema_name": {"unique_id": "macro.dbt.default__information_schema_name", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9613671}, "macro.dbt.list_schemas": {"unique_id": "macro.dbt.list_schemas", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.961557}, "macro.dbt.default__list_schemas": {"unique_id": "macro.dbt.default__list_schemas", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9618251}, "macro.dbt.check_schema_exists": {"unique_id": "macro.dbt.check_schema_exists", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9620512}, "macro.dbt.default__check_schema_exists": {"unique_id": "macro.dbt.default__check_schema_exists", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.962453}, "macro.dbt.list_relations_without_caching": {"unique_id": "macro.dbt.list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.962655}, "macro.dbt.default__list_relations_without_caching": {"unique_id": "macro.dbt.default__list_relations_without_caching", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "name": "default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9628239}, "macro.dbt.get_columns_in_relation": {"unique_id": "macro.dbt.get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.964653}, "macro.dbt.default__get_columns_in_relation": {"unique_id": "macro.dbt.default__get_columns_in_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.964823}, "macro.dbt.sql_convert_columns_in_relation": {"unique_id": "macro.dbt.sql_convert_columns_in_relation", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.965158}, "macro.dbt.get_columns_in_query": {"unique_id": "macro.dbt.get_columns_in_query", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9653602}, "macro.dbt.default__get_columns_in_query": {"unique_id": "macro.dbt.default__get_columns_in_query", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.96573}, "macro.dbt.alter_column_type": {"unique_id": "macro.dbt.alter_column_type", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9659781}, "macro.dbt.default__alter_column_type": {"unique_id": "macro.dbt.default__alter_column_type", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.966641}, "macro.dbt.alter_relation_add_remove_columns": {"unique_id": "macro.dbt.alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.966918}, "macro.dbt.default__alter_relation_add_remove_columns": {"unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "name": "default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.967777}, "macro.dbt.test_unique": {"unique_id": "macro.dbt.test_unique", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_bigquery.bigquery__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.968401}, "macro.dbt.test_not_null": {"unique_id": "macro.dbt.test_not_null", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9686542}, "macro.dbt.test_accepted_values": {"unique_id": "macro.dbt.test_accepted_values", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.968971}, "macro.dbt.test_relationships": {"unique_id": "macro.dbt.test_relationships", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "name": "test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.969268}, "macro.dbt_utils.except": {"unique_id": "macro.dbt_utils.except", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/except.sql", "original_file_path": "macros/cross_db_utils/except.sql", "name": "except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt_utils')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__except"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.96962}, "macro.dbt_utils.default__except": {"unique_id": "macro.dbt_utils.default__except", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/except.sql", "original_file_path": "macros/cross_db_utils/except.sql", "name": "default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.969698}, "macro.dbt_utils.bigquery__except": {"unique_id": "macro.dbt_utils.bigquery__except", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/except.sql", "original_file_path": "macros/cross_db_utils/except.sql", "name": "bigquery__except", "macro_sql": "{% macro bigquery__except() %}\n\n except distinct\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.96977}, "macro.dbt_utils.replace": {"unique_id": "macro.dbt_utils.replace", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/replace.sql", "original_file_path": "macros/cross_db_utils/replace.sql", "name": "replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9702091}, "macro.dbt_utils.default__replace": {"unique_id": "macro.dbt_utils.default__replace", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/replace.sql", "original_file_path": "macros/cross_db_utils/replace.sql", "name": "default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n \n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.970388}, "macro.dbt_utils.concat": {"unique_id": "macro.dbt_utils.concat", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/concat.sql", "original_file_path": "macros/cross_db_utils/concat.sql", "name": "concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__concat"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9707289}, "macro.dbt_utils.default__concat": {"unique_id": "macro.dbt_utils.default__concat", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/concat.sql", "original_file_path": "macros/cross_db_utils/concat.sql", "name": "default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.970934}, "macro.dbt_utils.type_string": {"unique_id": "macro.dbt_utils.type_string", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9717278}, "macro.dbt_utils.default__type_string": {"unique_id": "macro.dbt_utils.default__type_string", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "default__type_string", "macro_sql": "{% macro default__type_string() %}\n string\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.971807}, "macro.dbt_utils.redshift__type_string": {"unique_id": "macro.dbt_utils.redshift__type_string", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "redshift__type_string", "macro_sql": "\n\n{%- macro redshift__type_string() -%}\n varchar\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9718819}, "macro.dbt_utils.postgres__type_string": {"unique_id": "macro.dbt_utils.postgres__type_string", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "postgres__type_string", "macro_sql": "{% macro postgres__type_string() %}\n varchar\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.971956}, "macro.dbt_utils.snowflake__type_string": {"unique_id": "macro.dbt_utils.snowflake__type_string", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "snowflake__type_string", "macro_sql": "{% macro snowflake__type_string() %}\n varchar\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9720268}, "macro.dbt_utils.type_timestamp": {"unique_id": "macro.dbt_utils.type_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.972206}, "macro.dbt_utils.default__type_timestamp": {"unique_id": "macro.dbt_utils.default__type_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n timestamp\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.972283}, "macro.dbt_utils.postgres__type_timestamp": {"unique_id": "macro.dbt_utils.postgres__type_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "postgres__type_timestamp", "macro_sql": "{% macro postgres__type_timestamp() %}\n timestamp without time zone\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9723551}, "macro.dbt_utils.snowflake__type_timestamp": {"unique_id": "macro.dbt_utils.snowflake__type_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "snowflake__type_timestamp", "macro_sql": "{% macro snowflake__type_timestamp() %}\n timestamp_ntz\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.972427}, "macro.dbt_utils.type_float": {"unique_id": "macro.dbt_utils.type_float", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__type_float"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9725928}, "macro.dbt_utils.default__type_float": {"unique_id": "macro.dbt_utils.default__type_float", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "default__type_float", "macro_sql": "{% macro default__type_float() %}\n float\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.972668}, "macro.dbt_utils.bigquery__type_float": {"unique_id": "macro.dbt_utils.bigquery__type_float", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "bigquery__type_float", "macro_sql": "{% macro bigquery__type_float() %}\n float64\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.972737}, "macro.dbt_utils.type_numeric": {"unique_id": "macro.dbt_utils.type_numeric", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9729059}, "macro.dbt_utils.default__type_numeric": {"unique_id": "macro.dbt_utils.default__type_numeric", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n numeric(28, 6)\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.97298}, "macro.dbt_utils.bigquery__type_numeric": {"unique_id": "macro.dbt_utils.bigquery__type_numeric", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "bigquery__type_numeric", "macro_sql": "{% macro bigquery__type_numeric() %}\n numeric\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9730508}, "macro.dbt_utils.type_bigint": {"unique_id": "macro.dbt_utils.type_bigint", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.973218}, "macro.dbt_utils.default__type_bigint": {"unique_id": "macro.dbt_utils.default__type_bigint", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n bigint\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.97329}, "macro.dbt_utils.bigquery__type_bigint": {"unique_id": "macro.dbt_utils.bigquery__type_bigint", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "bigquery__type_bigint", "macro_sql": "{% macro bigquery__type_bigint() %}\n int64\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.973363}, "macro.dbt_utils.type_int": {"unique_id": "macro.dbt_utils.type_int", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.973587}, "macro.dbt_utils.default__type_int": {"unique_id": "macro.dbt_utils.default__type_int", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "default__type_int", "macro_sql": "{% macro default__type_int() %}\n int\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.973661}, "macro.dbt_utils.bigquery__type_int": {"unique_id": "macro.dbt_utils.bigquery__type_int", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datatypes.sql", "original_file_path": "macros/cross_db_utils/datatypes.sql", "name": "bigquery__type_int", "macro_sql": "{% macro bigquery__type_int() %}\n int64\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.973732}, "macro.dbt_utils._is_relation": {"unique_id": "macro.dbt_utils._is_relation", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/_is_relation.sql", "original_file_path": "macros/cross_db_utils/_is_relation.sql", "name": "_is_relation", "macro_sql": "{% macro _is_relation(obj, macro) %}\n {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%}\n {%- do exceptions.raise_compiler_error(\"Macro \" ~ macro ~ \" expected a Relation but received the value: \" ~ obj) -%}\n {%- endif -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9743009}, "macro.dbt_utils.length": {"unique_id": "macro.dbt_utils.length", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/length.sql", "original_file_path": "macros/cross_db_utils/length.sql", "name": "length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__length"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.974699}, "macro.dbt_utils.default__length": {"unique_id": "macro.dbt_utils.default__length", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/length.sql", "original_file_path": "macros/cross_db_utils/length.sql", "name": "default__length", "macro_sql": "{% macro default__length(expression) %}\n \n length(\n {{ expression }}\n )\n \n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9748101}, "macro.dbt_utils.redshift__length": {"unique_id": "macro.dbt_utils.redshift__length", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/length.sql", "original_file_path": "macros/cross_db_utils/length.sql", "name": "redshift__length", "macro_sql": "{% macro redshift__length(expression) %}\n\n len(\n {{ expression }}\n )\n \n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.974921}, "macro.dbt_utils.dateadd": {"unique_id": "macro.dbt_utils.dateadd", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/dateadd.sql", "original_file_path": "macros/cross_db_utils/dateadd.sql", "name": "dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.975585}, "macro.dbt_utils.default__dateadd": {"unique_id": "macro.dbt_utils.default__dateadd", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/dateadd.sql", "original_file_path": "macros/cross_db_utils/dateadd.sql", "name": "default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.975761}, "macro.dbt_utils.bigquery__dateadd": {"unique_id": "macro.dbt_utils.bigquery__dateadd", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/dateadd.sql", "original_file_path": "macros/cross_db_utils/dateadd.sql", "name": "bigquery__dateadd", "macro_sql": "{% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n datetime_add(\n cast( {{ from_date_or_timestamp }} as datetime),\n interval {{ interval }} {{ datepart }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9759371}, "macro.dbt_utils.postgres__dateadd": {"unique_id": "macro.dbt_utils.postgres__dateadd", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/dateadd.sql", "original_file_path": "macros/cross_db_utils/dateadd.sql", "name": "postgres__dateadd", "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.97611}, "macro.dbt_utils.redshift__dateadd": {"unique_id": "macro.dbt_utils.redshift__dateadd", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/dateadd.sql", "original_file_path": "macros/cross_db_utils/dateadd.sql", "name": "redshift__dateadd", "macro_sql": "{% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.976322}, "macro.dbt_utils.intersect": {"unique_id": "macro.dbt_utils.intersect", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/intersect.sql", "original_file_path": "macros/cross_db_utils/intersect.sql", "name": "intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__intersect"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.97667}, "macro.dbt_utils.default__intersect": {"unique_id": "macro.dbt_utils.default__intersect", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/intersect.sql", "original_file_path": "macros/cross_db_utils/intersect.sql", "name": "default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.976748}, "macro.dbt_utils.bigquery__intersect": {"unique_id": "macro.dbt_utils.bigquery__intersect", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/intersect.sql", "original_file_path": "macros/cross_db_utils/intersect.sql", "name": "bigquery__intersect", "macro_sql": "{% macro bigquery__intersect() %}\n\n intersect distinct\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.976822}, "macro.dbt_utils.escape_single_quotes": {"unique_id": "macro.dbt_utils.escape_single_quotes", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/escape_single_quotes.sql", "original_file_path": "macros/cross_db_utils/escape_single_quotes.sql", "name": "escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.977259}, "macro.dbt_utils.default__escape_single_quotes": {"unique_id": "macro.dbt_utils.default__escape_single_quotes", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/escape_single_quotes.sql", "original_file_path": "macros/cross_db_utils/escape_single_quotes.sql", "name": "default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.977414}, "macro.dbt_utils.snowflake__escape_single_quotes": {"unique_id": "macro.dbt_utils.snowflake__escape_single_quotes", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/escape_single_quotes.sql", "original_file_path": "macros/cross_db_utils/escape_single_quotes.sql", "name": "snowflake__escape_single_quotes", "macro_sql": "{% macro snowflake__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\", \"\\\\'\") }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.977558}, "macro.dbt_utils.bigquery__escape_single_quotes": {"unique_id": "macro.dbt_utils.bigquery__escape_single_quotes", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/escape_single_quotes.sql", "original_file_path": "macros/cross_db_utils/escape_single_quotes.sql", "name": "bigquery__escape_single_quotes", "macro_sql": "{% macro bigquery__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\", \"\\\\'\") }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.977699}, "macro.dbt_utils.right": {"unique_id": "macro.dbt_utils.right", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/right.sql", "original_file_path": "macros/cross_db_utils/right.sql", "name": "right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__right"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.978351}, "macro.dbt_utils.default__right": {"unique_id": "macro.dbt_utils.default__right", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/right.sql", "original_file_path": "macros/cross_db_utils/right.sql", "name": "default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n \n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.978496}, "macro.dbt_utils.bigquery__right": {"unique_id": "macro.dbt_utils.bigquery__right", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/right.sql", "original_file_path": "macros/cross_db_utils/right.sql", "name": "bigquery__right", "macro_sql": "{% macro bigquery__right(string_text, length_expression) %}\n\n case when {{ length_expression }} = 0 \n then ''\n else \n substr(\n {{ string_text }},\n -1 * ({{ length_expression }})\n )\n end\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9786618}, "macro.dbt_utils.snowflake__right": {"unique_id": "macro.dbt_utils.snowflake__right", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/right.sql", "original_file_path": "macros/cross_db_utils/right.sql", "name": "snowflake__right", "macro_sql": "{% macro snowflake__right(string_text, length_expression) %}\n\n case when {{ length_expression }} = 0 \n then ''\n else \n right(\n {{ string_text }},\n {{ length_expression }}\n )\n end\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.978902}, "macro.dbt_utils.listagg": {"unique_id": "macro.dbt_utils.listagg", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/listagg.sql", "original_file_path": "macros/cross_db_utils/listagg.sql", "name": "listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt_utils') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__listagg"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.981429}, "macro.dbt_utils.default__listagg": {"unique_id": "macro.dbt_utils.default__listagg", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/listagg.sql", "original_file_path": "macros/cross_db_utils/listagg.sql", "name": "default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.981877}, "macro.dbt_utils.bigquery__listagg": {"unique_id": "macro.dbt_utils.bigquery__listagg", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/listagg.sql", "original_file_path": "macros/cross_db_utils/listagg.sql", "name": "bigquery__listagg", "macro_sql": "{% macro bigquery__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n {% if limit_num -%}\n limit {{ limit_num }}\n {%- endif %}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.982191}, "macro.dbt_utils.postgres__listagg": {"unique_id": "macro.dbt_utils.postgres__listagg", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/listagg.sql", "original_file_path": "macros/cross_db_utils/listagg.sql", "name": "postgres__listagg", "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n \n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9826188}, "macro.dbt_utils.redshift__listagg": {"unique_id": "macro.dbt_utils.redshift__listagg", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/listagg.sql", "original_file_path": "macros/cross_db_utils/listagg.sql", "name": "redshift__listagg", "macro_sql": "{% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n {% set ns = namespace() %}\n {% set ns.delimiter_text_regex = delimiter_text|trim(\"'\") %}\n {% set special_chars %}\\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} \n {%- for char in special_chars.split(',') -%}\n {% set escape_char %}\\\\{{ char }}{% endset %}\n {% set ns.delimiter_text_regex = ns.delimiter_text_regex|replace(char,escape_char) %}\n {%- endfor -%}\n\n {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %}\n regexp_substr(\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,{{ regex }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.983659}, "macro.dbt_utils.datediff": {"unique_id": "macro.dbt_utils.datediff", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datediff.sql", "original_file_path": "macros/cross_db_utils/datediff.sql", "name": "datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.986515}, "macro.dbt_utils.default__datediff": {"unique_id": "macro.dbt_utils.default__datediff", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datediff.sql", "original_file_path": "macros/cross_db_utils/datediff.sql", "name": "default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9866931}, "macro.dbt_utils.bigquery__datediff": {"unique_id": "macro.dbt_utils.bigquery__datediff", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datediff.sql", "original_file_path": "macros/cross_db_utils/datediff.sql", "name": "bigquery__datediff", "macro_sql": "{% macro bigquery__datediff(first_date, second_date, datepart) -%}\n\n datetime_diff(\n cast({{second_date}} as datetime),\n cast({{first_date}} as datetime),\n {{datepart}}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.986867}, "macro.dbt_utils.postgres__datediff": {"unique_id": "macro.dbt_utils.postgres__datediff", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datediff.sql", "original_file_path": "macros/cross_db_utils/datediff.sql", "name": "postgres__datediff", "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9888818}, "macro.dbt_utils.redshift__datediff": {"unique_id": "macro.dbt_utils.redshift__datediff", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/datediff.sql", "original_file_path": "macros/cross_db_utils/datediff.sql", "name": "redshift__datediff", "macro_sql": "{% macro redshift__datediff(first_date, second_date, datepart) -%}\n\n {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9891021}, "macro.dbt_utils.safe_cast": {"unique_id": "macro.dbt_utils.safe_cast", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/safe_cast.sql", "original_file_path": "macros/cross_db_utils/safe_cast.sql", "name": "safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.989582}, "macro.dbt_utils.default__safe_cast": {"unique_id": "macro.dbt_utils.default__safe_cast", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/safe_cast.sql", "original_file_path": "macros/cross_db_utils/safe_cast.sql", "name": "default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.989731}, "macro.dbt_utils.snowflake__safe_cast": {"unique_id": "macro.dbt_utils.snowflake__safe_cast", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/safe_cast.sql", "original_file_path": "macros/cross_db_utils/safe_cast.sql", "name": "snowflake__safe_cast", "macro_sql": "{% macro snowflake__safe_cast(field, type) %}\n try_cast({{field}} as {{type}})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.989866}, "macro.dbt_utils.bigquery__safe_cast": {"unique_id": "macro.dbt_utils.bigquery__safe_cast", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/safe_cast.sql", "original_file_path": "macros/cross_db_utils/safe_cast.sql", "name": "bigquery__safe_cast", "macro_sql": "{% macro bigquery__safe_cast(field, type) %}\n safe_cast({{field}} as {{type}})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.990002}, "macro.dbt_utils.hash": {"unique_id": "macro.dbt_utils.hash", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/hash.sql", "original_file_path": "macros/cross_db_utils/hash.sql", "name": "hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__hash"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.990392}, "macro.dbt_utils.default__hash": {"unique_id": "macro.dbt_utils.default__hash", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/hash.sql", "original_file_path": "macros/cross_db_utils/hash.sql", "name": "default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{field}} as {{dbt_utils.type_string()}}))\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.990537}, "macro.dbt_utils.bigquery__hash": {"unique_id": "macro.dbt_utils.bigquery__hash", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/hash.sql", "original_file_path": "macros/cross_db_utils/hash.sql", "name": "bigquery__hash", "macro_sql": "{% macro bigquery__hash(field) -%}\n to_hex({{dbt_utils.default__hash(field)}})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9906719}, "macro.dbt_utils.cast_bool_to_text": {"unique_id": "macro.dbt_utils.cast_bool_to_text", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/cast_bool_to_text.sql", "original_file_path": "macros/cross_db_utils/cast_bool_to_text.sql", "name": "cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.991076}, "macro.dbt_utils.default__cast_bool_to_text": {"unique_id": "macro.dbt_utils.default__cast_bool_to_text", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/cast_bool_to_text.sql", "original_file_path": "macros/cross_db_utils/cast_bool_to_text.sql", "name": "default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ dbt_utils.type_string() }})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.991227}, "macro.dbt_utils.redshift__cast_bool_to_text": {"unique_id": "macro.dbt_utils.redshift__cast_bool_to_text", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/cast_bool_to_text.sql", "original_file_path": "macros/cross_db_utils/cast_bool_to_text.sql", "name": "redshift__cast_bool_to_text", "macro_sql": "{% macro redshift__cast_bool_to_text(field) %}\n case\n when {{ field }} is true then 'true'\n when {{ field }} is false then 'false'\n end::text\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.991358}, "macro.dbt_utils.identifier": {"unique_id": "macro.dbt_utils.identifier", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/identifier.sql", "original_file_path": "macros/cross_db_utils/identifier.sql", "name": "identifier", "macro_sql": "{% macro identifier(value) %}\t\n {%- set error_message = '\n Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \\\n Use `adapter.quote` instead. The {}.{} model triggered this warning. \\\n '.format(model.package_name, model.name) -%}\n {%- do exceptions.warn(error_message) -%}\n {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__identifier"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9919329}, "macro.dbt_utils.default__identifier": {"unique_id": "macro.dbt_utils.default__identifier", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/identifier.sql", "original_file_path": "macros/cross_db_utils/identifier.sql", "name": "default__identifier", "macro_sql": "{% macro default__identifier(value) -%}\t\n \"{{ value }}\"\t\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.992047}, "macro.dbt_utils.bigquery__identifier": {"unique_id": "macro.dbt_utils.bigquery__identifier", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/identifier.sql", "original_file_path": "macros/cross_db_utils/identifier.sql", "name": "bigquery__identifier", "macro_sql": "{% macro bigquery__identifier(value) -%}\t\n `{{ value }}`\t\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.992157}, "macro.dbt_utils.any_value": {"unique_id": "macro.dbt_utils.any_value", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/any_value.sql", "original_file_path": "macros/cross_db_utils/any_value.sql", "name": "any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__any_value"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9925508}, "macro.dbt_utils.default__any_value": {"unique_id": "macro.dbt_utils.default__any_value", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/any_value.sql", "original_file_path": "macros/cross_db_utils/any_value.sql", "name": "default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n \n any_value({{ expression }})\n \n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.992662}, "macro.dbt_utils.postgres__any_value": {"unique_id": "macro.dbt_utils.postgres__any_value", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/any_value.sql", "original_file_path": "macros/cross_db_utils/any_value.sql", "name": "postgres__any_value", "macro_sql": "{% macro postgres__any_value(expression) -%}\n {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#}\n min({{ expression }})\n \n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.992772}, "macro.dbt_utils.position": {"unique_id": "macro.dbt_utils.position", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/position.sql", "original_file_path": "macros/cross_db_utils/position.sql", "name": "position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__position"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.99323}, "macro.dbt_utils.default__position": {"unique_id": "macro.dbt_utils.default__position", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/position.sql", "original_file_path": "macros/cross_db_utils/position.sql", "name": "default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n \n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.993373}, "macro.dbt_utils.bigquery__position": {"unique_id": "macro.dbt_utils.bigquery__position", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/position.sql", "original_file_path": "macros/cross_db_utils/position.sql", "name": "bigquery__position", "macro_sql": "{% macro bigquery__position(substring_text, string_text) %}\n\n strpos(\n {{ string_text }},\n {{ substring_text }}\n \n )\n \n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9935129}, "macro.dbt_utils.string_literal": {"unique_id": "macro.dbt_utils.string_literal", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/literal.sql", "original_file_path": "macros/cross_db_utils/literal.sql", "name": "string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.993919}, "macro.dbt_utils.default__string_literal": {"unique_id": "macro.dbt_utils.default__string_literal", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/literal.sql", "original_file_path": "macros/cross_db_utils/literal.sql", "name": "default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.994026}, "macro.dbt_utils.current_timestamp": {"unique_id": "macro.dbt_utils.current_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "current_timestamp", "macro_sql": "{% macro current_timestamp() -%}\n {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.994685}, "macro.dbt_utils.default__current_timestamp": {"unique_id": "macro.dbt_utils.default__current_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() %}\n current_timestamp::{{dbt_utils.type_timestamp()}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.994803}, "macro.dbt_utils.redshift__current_timestamp": {"unique_id": "macro.dbt_utils.redshift__current_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "redshift__current_timestamp", "macro_sql": "{% macro redshift__current_timestamp() %}\n getdate()\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.994876}, "macro.dbt_utils.bigquery__current_timestamp": {"unique_id": "macro.dbt_utils.bigquery__current_timestamp", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "bigquery__current_timestamp", "macro_sql": "{% macro bigquery__current_timestamp() %}\n current_timestamp\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9949498}, "macro.dbt_utils.current_timestamp_in_utc": {"unique_id": "macro.dbt_utils.current_timestamp_in_utc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "current_timestamp_in_utc", "macro_sql": "{% macro current_timestamp_in_utc() -%}\n {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__current_timestamp_in_utc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9951181}, "macro.dbt_utils.default__current_timestamp_in_utc": {"unique_id": "macro.dbt_utils.default__current_timestamp_in_utc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "default__current_timestamp_in_utc", "macro_sql": "{% macro default__current_timestamp_in_utc() %}\n {{dbt_utils.current_timestamp()}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9952362}, "macro.dbt_utils.snowflake__current_timestamp_in_utc": {"unique_id": "macro.dbt_utils.snowflake__current_timestamp_in_utc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "snowflake__current_timestamp_in_utc", "macro_sql": "{% macro snowflake__current_timestamp_in_utc() %}\n convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.current_timestamp", "macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.99539}, "macro.dbt_utils.postgres__current_timestamp_in_utc": {"unique_id": "macro.dbt_utils.postgres__current_timestamp_in_utc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "postgres__current_timestamp_in_utc", "macro_sql": "{% macro postgres__current_timestamp_in_utc() %}\n (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.995511}, "macro.dbt_utils.redshift__current_timestamp_in_utc": {"unique_id": "macro.dbt_utils.redshift__current_timestamp_in_utc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/cross_db_utils/current_timestamp.sql", "name": "redshift__current_timestamp_in_utc", "macro_sql": "{% macro redshift__current_timestamp_in_utc() %}\n {{ return(dbt_utils.default__current_timestamp_in_utc()) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__current_timestamp_in_utc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.995651}, "macro.dbt_utils.width_bucket": {"unique_id": "macro.dbt_utils.width_bucket", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/width_bucket.sql", "original_file_path": "macros/cross_db_utils/width_bucket.sql", "name": "width_bucket", "macro_sql": "{% macro width_bucket(expr, min_value, max_value, num_buckets) %}\n {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__width_bucket"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.997227}, "macro.dbt_utils.default__width_bucket": {"unique_id": "macro.dbt_utils.default__width_bucket", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/width_bucket.sql", "original_file_path": "macros/cross_db_utils/width_bucket.sql", "name": "default__width_bucket", "macro_sql": "{% macro default__width_bucket(expr, min_value, max_value, num_buckets) -%}\n\n {% set bin_size -%}\n (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} )\n {%- endset %}\n (\n -- to break ties when the amount is eaxtly at the bucket egde\n case\n when\n mod(\n {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }},\n {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }}\n ) = 0\n then 1\n else 0\n end\n ) +\n -- Anything over max_value goes the N+1 bucket\n least(\n ceil(\n ({{ expr }} - {{ min_value }})/{{ bin_size }}\n ),\n {{ num_buckets }} + 1\n )\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.safe_cast", "macro.dbt_utils.type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9977422}, "macro.dbt_utils.redshift__width_bucket": {"unique_id": "macro.dbt_utils.redshift__width_bucket", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/width_bucket.sql", "original_file_path": "macros/cross_db_utils/width_bucket.sql", "name": "redshift__width_bucket", "macro_sql": "{% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%}\n\n {% set bin_size -%}\n (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} )\n {%- endset %}\n (\n -- to break ties when the amount is exactly at the bucket edge\n case\n when\n {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} %\n {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }}\n = 0\n then 1\n else 0\n end\n ) +\n -- Anything over max_value goes the N+1 bucket\n least(\n ceil(\n ({{ expr }} - {{ min_value }})/{{ bin_size }}\n ),\n {{ num_buckets }} + 1\n )\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.safe_cast", "macro.dbt_utils.type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.998252}, "macro.dbt_utils.snowflake__width_bucket": {"unique_id": "macro.dbt_utils.snowflake__width_bucket", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/width_bucket.sql", "original_file_path": "macros/cross_db_utils/width_bucket.sql", "name": "snowflake__width_bucket", "macro_sql": "{% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %}\n width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} )\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9984581}, "macro.dbt_utils.bool_or": {"unique_id": "macro.dbt_utils.bool_or", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/bool_or.sql", "original_file_path": "macros/cross_db_utils/bool_or.sql", "name": "bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.998893}, "macro.dbt_utils.default__bool_or": {"unique_id": "macro.dbt_utils.default__bool_or", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/bool_or.sql", "original_file_path": "macros/cross_db_utils/bool_or.sql", "name": "default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n \n bool_or({{ expression }})\n \n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.999003}, "macro.dbt_utils.snowflake__bool_or": {"unique_id": "macro.dbt_utils.snowflake__bool_or", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/bool_or.sql", "original_file_path": "macros/cross_db_utils/bool_or.sql", "name": "snowflake__bool_or", "macro_sql": "{% macro snowflake__bool_or(expression) -%}\n \n boolor_agg({{ expression }})\n \n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.9991071}, "macro.dbt_utils.bigquery__bool_or": {"unique_id": "macro.dbt_utils.bigquery__bool_or", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/bool_or.sql", "original_file_path": "macros/cross_db_utils/bool_or.sql", "name": "bigquery__bool_or", "macro_sql": "{% macro bigquery__bool_or(expression) -%}\n \n logical_or({{ expression }})\n \n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.99921}, "macro.dbt_utils.last_day": {"unique_id": "macro.dbt_utils.last_day", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/last_day.sql", "original_file_path": "macros/cross_db_utils/last_day.sql", "name": "last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986670.999879}, "macro.dbt_utils.default_last_day": {"unique_id": "macro.dbt_utils.default_last_day", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/last_day.sql", "original_file_path": "macros/cross_db_utils/last_day.sql", "name": "default_last_day", "macro_sql": "\n\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt_utils.dateadd('day', '-1',\n dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.dateadd", "macro.dbt_utils.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.000232}, "macro.dbt_utils.default__last_day": {"unique_id": "macro.dbt_utils.default__last_day", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/last_day.sql", "original_file_path": "macros/cross_db_utils/last_day.sql", "name": "default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt_utils.default_last_day(date, datepart)}}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0003939}, "macro.dbt_utils.postgres__last_day": {"unique_id": "macro.dbt_utils.postgres__last_day", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/last_day.sql", "original_file_path": "macros/cross_db_utils/last_day.sql", "name": "postgres__last_day", "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt_utils.dateadd('day', '-1',\n dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt_utils.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.dateadd", "macro.dbt_utils.date_trunc", "macro.dbt_utils.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.000822}, "macro.dbt_utils.redshift__last_day": {"unique_id": "macro.dbt_utils.redshift__last_day", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/last_day.sql", "original_file_path": "macros/cross_db_utils/last_day.sql", "name": "redshift__last_day", "macro_sql": "{% macro redshift__last_day(date, datepart) %}\n\n {{ return(dbt_utils.default__last_day(date, datepart)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.001011}, "macro.dbt_utils.split_part": {"unique_id": "macro.dbt_utils.split_part", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/split_part.sql", "original_file_path": "macros/cross_db_utils/split_part.sql", "name": "split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__split_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.002656}, "macro.dbt_utils.default__split_part": {"unique_id": "macro.dbt_utils.default__split_part", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/split_part.sql", "original_file_path": "macros/cross_db_utils/split_part.sql", "name": "default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0028338}, "macro.dbt_utils._split_part_negative": {"unique_id": "macro.dbt_utils._split_part_negative", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/split_part.sql", "original_file_path": "macros/cross_db_utils/split_part.sql", "name": "_split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }}) \n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0030751}, "macro.dbt_utils.postgres__split_part": {"unique_id": "macro.dbt_utils.postgres__split_part", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/split_part.sql", "original_file_path": "macros/cross_db_utils/split_part.sql", "name": "postgres__split_part", "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt_utils.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt_utils._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__split_part", "macro.dbt_utils._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.003443}, "macro.dbt_utils.redshift__split_part": {"unique_id": "macro.dbt_utils.redshift__split_part", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/split_part.sql", "original_file_path": "macros/cross_db_utils/split_part.sql", "name": "redshift__split_part", "macro_sql": "{% macro redshift__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt_utils.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt_utils._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__split_part", "macro.dbt_utils._split_part_negative"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0038111}, "macro.dbt_utils.bigquery__split_part": {"unique_id": "macro.dbt_utils.bigquery__split_part", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/split_part.sql", "original_file_path": "macros/cross_db_utils/split_part.sql", "name": "bigquery__split_part", "macro_sql": "{% macro bigquery__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n split(\n {{ string_text }},\n {{ delimiter_text }}\n )[safe_offset({{ part_number - 1 }})]\n {% else %}\n split(\n {{ string_text }},\n {{ delimiter_text }}\n )[safe_offset(\n length({{ string_text }}) \n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1\n )]\n {% endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.004208}, "macro.dbt_utils.date_trunc": {"unique_id": "macro.dbt_utils.date_trunc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/date_trunc.sql", "original_file_path": "macros/cross_db_utils/date_trunc.sql", "name": "date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.004648}, "macro.dbt_utils.default__date_trunc": {"unique_id": "macro.dbt_utils.default__date_trunc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/date_trunc.sql", "original_file_path": "macros/cross_db_utils/date_trunc.sql", "name": "default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.004786}, "macro.dbt_utils.bigquery__date_trunc": {"unique_id": "macro.dbt_utils.bigquery__date_trunc", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/date_trunc.sql", "original_file_path": "macros/cross_db_utils/date_trunc.sql", "name": "bigquery__date_trunc", "macro_sql": "{% macro bigquery__date_trunc(datepart, date) -%}\n timestamp_trunc(\n cast({{date}} as timestamp),\n {{datepart}}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0049238}, "macro.dbt_utils._is_ephemeral": {"unique_id": "macro.dbt_utils._is_ephemeral", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/cross_db_utils/_is_ephemeral.sql", "original_file_path": "macros/cross_db_utils/_is_ephemeral.sql", "name": "_is_ephemeral", "macro_sql": "{% macro _is_ephemeral(obj, macro) %}\n {%- if obj.is_cte -%}\n {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %}\n {% if obj.name.startswith(ephemeral_prefix) %}\n {% set model_name = obj.name[(ephemeral_prefix|length):] %}\n {% else %}\n {% set model_name = obj.name %}\n {%- endif -%}\n {% set error_message %}\nThe `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema.\n\n`{{ model_name }}` is an ephemeral model. Consider making it a view or table instead.\n {% endset %}\n {%- do exceptions.raise_compiler_error(error_message) -%}\n {%- endif -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.005897}, "macro.dbt_utils.get_period_boundaries": {"unique_id": "macro.dbt_utils.get_period_boundaries", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/materializations/insert_by_period_materialization.sql", "original_file_path": "macros/materializations/insert_by_period_materialization.sql", "name": "get_period_boundaries", "macro_sql": "{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}\n {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_period_boundaries"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0122309}, "macro.dbt_utils.default__get_period_boundaries": {"unique_id": "macro.dbt_utils.default__get_period_boundaries", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/materializations/insert_by_period_materialization.sql", "original_file_path": "macros/materializations/insert_by_period_materialization.sql", "name": "default__get_period_boundaries", "macro_sql": "{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}\n\n {% call statement('period_boundaries', fetch_result=True) -%}\n with data as (\n select\n coalesce(max(\"{{timestamp_field}}\"), '{{start_date}}')::timestamp as start_timestamp,\n coalesce(\n {{dbt_utils.dateadd('millisecond',\n -1,\n \"nullif('\" ~ stop_date ~ \"','')::timestamp\")}},\n {{dbt_utils.current_timestamp()}}\n ) as stop_timestamp\n from \"{{target_schema}}\".\"{{target_table}}\"\n )\n\n select\n start_timestamp,\n stop_timestamp,\n {{dbt_utils.datediff('start_timestamp',\n 'stop_timestamp',\n period)}} + 1 as num_periods\n from data\n {%- endcall %}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_utils.dateadd", "macro.dbt_utils.current_timestamp", "macro.dbt_utils.datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.012799}, "macro.dbt_utils.get_period_sql": {"unique_id": "macro.dbt_utils.get_period_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/materializations/insert_by_period_materialization.sql", "original_file_path": "macros/materializations/insert_by_period_materialization.sql", "name": "get_period_sql", "macro_sql": "{% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}\n {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_period_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0131462}, "macro.dbt_utils.default__get_period_sql": {"unique_id": "macro.dbt_utils.default__get_period_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/materializations/insert_by_period_materialization.sql", "original_file_path": "macros/materializations/insert_by_period_materialization.sql", "name": "default__get_period_sql", "macro_sql": "{% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}\n\n {%- set period_filter -%}\n (\"{{timestamp_field}}\" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and\n \"{{timestamp_field}}\" <= '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' + interval '1 {{period}}' and\n \"{{timestamp_field}}\" < '{{stop_timestamp}}'::timestamp)\n {%- endset -%}\n\n {%- set filtered_sql = sql | replace(\"__PERIOD_FILTER__\", period_filter) -%}\n\n select\n {{target_cols_csv}}\n from (\n {{filtered_sql}}\n )\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.013686}, "macro.dbt_utils.materialization_insert_by_period_default": {"unique_id": "macro.dbt_utils.materialization_insert_by_period_default", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/materializations/insert_by_period_materialization.sql", "original_file_path": "macros/materializations/insert_by_period_materialization.sql", "name": "materialization_insert_by_period_default", "macro_sql": "{% materialization insert_by_period, default -%}\n {%- set timestamp_field = config.require('timestamp_field') -%}\n {%- set start_date = config.require('start_date') -%}\n {%- set stop_date = config.get('stop_date') or '' -%}\n {%- set period = config.get('period') or 'week' -%}\n\n {%- if sql.find('__PERIOD_FILTER__') == -1 -%}\n {%- set error_message -%}\n Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql\n {%- endset -%}\n {{ exceptions.raise_compiler_error(error_message) }}\n {%- endif -%}\n\n {%- set identifier = model['name'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%}\n\n {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}\n {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}\n\n {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%}\n {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%}\n {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%}\n\n -- setup\n {% if old_relation is none -%}\n -- noop\n {%- elif should_truncate -%}\n {{adapter.truncate_relation(old_relation)}}\n {%- elif should_drop -%}\n {{adapter.drop_relation(old_relation)}}\n {%- set old_relation = none -%}\n {%- endif %}\n\n {{run_hooks(pre_hooks, inside_transaction=False)}}\n\n -- `begin` happens here, so `commit` after it to finish the transaction\n {{run_hooks(pre_hooks, inside_transaction=True)}}\n {% call statement() -%}\n begin; -- make extra sure we've closed out the transaction\n commit;\n {%- endcall %}\n\n -- build model\n {% if force_create or old_relation is none -%}\n {# Create an empty target table -#}\n {% call statement('main') -%}\n {%- set empty_sql = sql | replace(\"__PERIOD_FILTER__\", 'false') -%}\n {{create_table_as(False, target_relation, empty_sql)}}\n {%- endcall %}\n {%- endif %}\n\n {% set _ = dbt_utils.get_period_boundaries(schema,\n identifier,\n timestamp_field,\n start_date,\n stop_date,\n period) %}\n {%- set start_timestamp = load_result('period_boundaries')['data'][0][0] | string -%}\n {%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%}\n {%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%}\n\n {% set target_columns = adapter.get_columns_in_relation(target_relation) %}\n {%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%}\n {%- set loop_vars = {'sum_rows_inserted': 0} -%}\n\n -- commit each period as a separate transaction\n {% for i in range(num_periods) -%}\n {%- set msg = \"Running for \" ~ period ~ \" \" ~ (i + 1) ~ \" of \" ~ (num_periods) -%}\n {{ dbt_utils.log_info(msg) }}\n\n {%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, type='table') -%}\n {% call statement() -%}\n {% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv,\n sql,\n timestamp_field,\n period,\n start_timestamp,\n stop_timestamp,\n i) %}\n {{dbt.create_table_as(True, tmp_relation, tmp_table_sql)}}\n {%- endcall %}\n\n {{adapter.expand_target_column_types(from_relation=tmp_relation,\n to_relation=target_relation)}}\n {%- set name = 'main-' ~ i -%}\n {% call statement(name, fetch_result=True) -%}\n insert into {{target_relation}} ({{target_cols_csv}})\n (\n select\n {{target_cols_csv}}\n from {{tmp_relation.include(schema=False)}}\n );\n {%- endcall %}\n {% set result = load_result('main-' ~ i) %}\n {% if 'response' in result.keys() %} {# added in v0.19.0 #}\n {% set rows_inserted = result['response']['rows_affected'] %}\n {% else %} {# older versions #}\n {% set rows_inserted = result['status'].split(\" \")[2] | int %}\n {% endif %}\n \n {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%}\n {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%}\n\n {%- set msg = \"Ran for \" ~ period ~ \" \" ~ (i + 1) ~ \" of \" ~ (num_periods) ~ \"; \" ~ rows_inserted ~ \" records inserted\" -%}\n {{ dbt_utils.log_info(msg) }}\n\n {%- endfor %}\n\n {% call statement() -%}\n begin;\n {%- endcall %}\n\n {{run_hooks(post_hooks, inside_transaction=True)}}\n\n {% call statement() -%}\n commit;\n {%- endcall %}\n\n {{run_hooks(post_hooks, inside_transaction=False)}}\n\n {%- set status_string = \"INSERT \" ~ loop_vars['sum_rows_inserted'] -%}\n\n {% call noop_statement('main', status_string) -%}\n -- no-op\n {%- endcall %}\n\n -- Return the relations created in this materialization\n {{ return({'relations': [target_relation]}) }} \n\n{%- endmaterialization %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt_utils.get_period_boundaries", "macro.dbt_utils.log_info", "macro.dbt_utils.get_period_sql", "macro.dbt.noop_statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0194829}, "macro.dbt_utils.get_url_host": {"unique_id": "macro.dbt_utils.get_url_host", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/web/get_url_host.sql", "original_file_path": "macros/web/get_url_host.sql", "name": "get_url_host", "macro_sql": "{% macro get_url_host(field) -%}\n {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_url_host"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.020003}, "macro.dbt_utils.default__get_url_host": {"unique_id": "macro.dbt_utils.default__get_url_host", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/web/get_url_host.sql", "original_file_path": "macros/web/get_url_host.sql", "name": "default__get_url_host", "macro_sql": "{% macro default__get_url_host(field) -%}\n\n{%- set parsed =\n dbt_utils.split_part(\n dbt_utils.split_part(\n dbt_utils.replace(\n dbt_utils.replace(\n dbt_utils.replace(field, \"'android-app://'\", \"''\"\n ), \"'http://'\", \"''\"\n ), \"'https://'\", \"''\"\n ), \"'/'\", 1\n ), \"'?'\", 1\n )\n\n-%}\n\n\n {{ dbt_utils.safe_cast(\n parsed,\n dbt_utils.type_string()\n )}}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.split_part", "macro.dbt_utils.replace", "macro.dbt_utils.safe_cast", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.020545}, "macro.dbt_utils.get_url_path": {"unique_id": "macro.dbt_utils.get_url_path", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/web/get_url_path.sql", "original_file_path": "macros/web/get_url_path.sql", "name": "get_url_path", "macro_sql": "{% macro get_url_path(field) -%}\n {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_url_path"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.021155}, "macro.dbt_utils.default__get_url_path": {"unique_id": "macro.dbt_utils.default__get_url_path", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/web/get_url_path.sql", "original_file_path": "macros/web/get_url_path.sql", "name": "default__get_url_path", "macro_sql": "{% macro default__get_url_path(field) -%}\n\n {%- set stripped_url = \n dbt_utils.replace(\n dbt_utils.replace(field, \"'http://'\", \"''\"), \"'https://'\", \"''\")\n -%}\n\n {%- set first_slash_pos -%}\n coalesce(\n nullif({{dbt_utils.position(\"'/'\", stripped_url)}}, 0),\n {{dbt_utils.position(\"'?'\", stripped_url)}} - 1\n )\n {%- endset -%}\n\n {%- set parsed_path =\n dbt_utils.split_part(\n dbt_utils.right(\n stripped_url, \n dbt_utils.length(stripped_url) ~ \"-\" ~ first_slash_pos\n ), \n \"'?'\", 1\n )\n -%}\n\n {{ dbt_utils.safe_cast(\n parsed_path,\n dbt_utils.type_string()\n )}}\n \n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.replace", "macro.dbt_utils.position", "macro.dbt_utils.split_part", "macro.dbt_utils.right", "macro.dbt_utils.length", "macro.dbt_utils.safe_cast", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.021878}, "macro.dbt_utils.get_url_parameter": {"unique_id": "macro.dbt_utils.get_url_parameter", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/web/get_url_parameter.sql", "original_file_path": "macros/web/get_url_parameter.sql", "name": "get_url_parameter", "macro_sql": "{% macro get_url_parameter(field, url_parameter) -%}\n {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_url_parameter"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.022306}, "macro.dbt_utils.default__get_url_parameter": {"unique_id": "macro.dbt_utils.default__get_url_parameter", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/web/get_url_parameter.sql", "original_file_path": "macros/web/get_url_parameter.sql", "name": "default__get_url_parameter", "macro_sql": "{% macro default__get_url_parameter(field, url_parameter) -%}\n\n{%- set formatted_url_parameter = \"'\" + url_parameter + \"='\" -%}\n\n{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), \"'&'\", 1) -%}\n\nnullif({{ split }},'')\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.split_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.02266}, "macro.dbt_utils.test_fewer_rows_than": {"unique_id": "macro.dbt_utils.test_fewer_rows_than", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/fewer_rows_than.sql", "original_file_path": "macros/generic_tests/fewer_rows_than.sql", "name": "test_fewer_rows_than", "macro_sql": "{% test fewer_rows_than(model, compare_model) %}\n {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_fewer_rows_than"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.023298}, "macro.dbt_utils.default__test_fewer_rows_than": {"unique_id": "macro.dbt_utils.default__test_fewer_rows_than", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/fewer_rows_than.sql", "original_file_path": "macros/generic_tests/fewer_rows_than.sql", "name": "default__test_fewer_rows_than", "macro_sql": "{% macro default__test_fewer_rows_than(model, compare_model) %}\n\n{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }}\n\nwith a as (\n\n select count(*) as count_our_model from {{ model }}\n\n),\nb as (\n\n select count(*) as count_comparison_model from {{ compare_model }}\n\n),\ncounts as (\n\n select\n count_our_model,\n count_comparison_model\n from a\n cross join b\n\n),\nfinal as (\n\n select *,\n case\n -- fail the test if we have more rows than the reference model and return the row count delta\n when count_our_model > count_comparison_model then (count_our_model - count_comparison_model)\n -- fail the test if they are the same number\n when count_our_model = count_comparison_model then 1\n -- pass the test if the delta is positive (i.e. return the number 0)\n else 0\n end as row_count_delta\n from counts\n\n)\n\nselect * from final\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.023542}, "macro.dbt_utils.test_equal_rowcount": {"unique_id": "macro.dbt_utils.test_equal_rowcount", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/equal_rowcount.sql", "original_file_path": "macros/generic_tests/equal_rowcount.sql", "name": "test_equal_rowcount", "macro_sql": "{% test equal_rowcount(model, compare_model) %}\n {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_equal_rowcount"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.024033}, "macro.dbt_utils.default__test_equal_rowcount": {"unique_id": "macro.dbt_utils.default__test_equal_rowcount", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/equal_rowcount.sql", "original_file_path": "macros/generic_tests/equal_rowcount.sql", "name": "default__test_equal_rowcount", "macro_sql": "{% macro default__test_equal_rowcount(model, compare_model) %}\n\n{#-- Needs to be set at parse time, before we return '' below --#}\n{{ config(fail_calc = 'coalesce(diff_count, 0)') }}\n\n{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}\n{%- if not execute -%}\n {{ return('') }}\n{% endif %}\n\nwith a as (\n\n select count(*) as count_a from {{ model }}\n\n),\nb as (\n\n select count(*) as count_b from {{ compare_model }}\n\n),\nfinal as (\n\n select\n count_a,\n count_b,\n abs(count_a - count_b) as diff_count\n from a\n cross join b\n\n)\n\nselect * from final\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0243578}, "macro.dbt_utils.test_relationships_where": {"unique_id": "macro.dbt_utils.test_relationships_where", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/relationships_where.sql", "original_file_path": "macros/generic_tests/relationships_where.sql", "name": "test_relationships_where", "macro_sql": "{% test relationships_where(model, column_name, to, field, from_condition=\"1=1\", to_condition=\"1=1\") %}\n {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_relationships_where"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.025119}, "macro.dbt_utils.default__test_relationships_where": {"unique_id": "macro.dbt_utils.default__test_relationships_where", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/relationships_where.sql", "original_file_path": "macros/generic_tests/relationships_where.sql", "name": "default__test_relationships_where", "macro_sql": "{% macro default__test_relationships_where(model, column_name, to, field, from_condition=\"1=1\", to_condition=\"1=1\") %}\n\n{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #}\n{# ref https://stackoverflow.com/a/7170753/3842610 #}\n\nwith left_table as (\n\n select\n {{column_name}} as id\n\n from {{model}}\n\n where {{column_name}} is not null\n and {{from_condition}}\n\n),\n\nright_table as (\n\n select\n {{field}} as id\n\n from {{to}}\n\n where {{field}} is not null\n and {{to_condition}}\n\n),\n\nexceptions as (\n\n select\n left_table.id,\n right_table.id as right_id\n\n from left_table\n\n left join right_table\n on left_table.id = right_table.id\n\n where right_table.id is null\n\n)\n\nselect * from exceptions\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.025501}, "macro.dbt_utils.test_recency": {"unique_id": "macro.dbt_utils.test_recency", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/recency.sql", "original_file_path": "macros/generic_tests/recency.sql", "name": "test_recency", "macro_sql": "{% test recency(model, field, datepart, interval) %}\n {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_recency"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0260081}, "macro.dbt_utils.default__test_recency": {"unique_id": "macro.dbt_utils.default__test_recency", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/recency.sql", "original_file_path": "macros/generic_tests/recency.sql", "name": "default__test_recency", "macro_sql": "{% macro default__test_recency(model, field, datepart, interval) %}\n\n{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %}\n\nwith recency as (\n\n select max({{field}}) as most_recent\n from {{ model }}\n\n)\n\nselect\n\n most_recent,\n {{ threshold }} as threshold\n\nfrom recency\nwhere most_recent < {{ threshold }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.dateadd", "macro.dbt_utils.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.02637}, "macro.dbt_utils.test_not_constant": {"unique_id": "macro.dbt_utils.test_not_constant", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/not_constant.sql", "original_file_path": "macros/generic_tests/not_constant.sql", "name": "test_not_constant", "macro_sql": "{% test not_constant(model, column_name) %}\n {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_not_constant"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.026787}, "macro.dbt_utils.default__test_not_constant": {"unique_id": "macro.dbt_utils.default__test_not_constant", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/not_constant.sql", "original_file_path": "macros/generic_tests/not_constant.sql", "name": "default__test_not_constant", "macro_sql": "{% macro default__test_not_constant(model, column_name) %}\n\n\nselect\n {# In TSQL, subquery aggregate columns need aliases #}\n {# thus: a filler col name, 'filler_column' #}\n count(distinct {{ column_name }}) as filler_column\n\nfrom {{ model }}\n\nhaving count(distinct {{ column_name }}) = 1\n\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.026976}, "macro.dbt_utils.test_accepted_range": {"unique_id": "macro.dbt_utils.test_accepted_range", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/accepted_range.sql", "original_file_path": "macros/generic_tests/accepted_range.sql", "name": "test_accepted_range", "macro_sql": "{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %}\n {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_accepted_range"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0276978}, "macro.dbt_utils.default__test_accepted_range": {"unique_id": "macro.dbt_utils.default__test_accepted_range", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/accepted_range.sql", "original_file_path": "macros/generic_tests/accepted_range.sql", "name": "default__test_accepted_range", "macro_sql": "{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %}\n\nwith meet_condition as(\n select *\n from {{ model }}\n),\n\nvalidation_errors as (\n select *\n from meet_condition\n where\n -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds\n 1 = 2\n\n {%- if min_value is not none %}\n -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule.\n or not {{ column_name }} > {{- \"=\" if inclusive }} {{ min_value }}\n {%- endif %}\n\n {%- if max_value is not none %}\n -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule.\n or not {{ column_name }} < {{- \"=\" if inclusive }} {{ max_value }}\n {%- endif %}\n)\n\nselect *\nfrom validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.051987}, "macro.dbt_utils.test_not_accepted_values": {"unique_id": "macro.dbt_utils.test_not_accepted_values", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/not_accepted_values.sql", "original_file_path": "macros/generic_tests/not_accepted_values.sql", "name": "test_not_accepted_values", "macro_sql": "{% test not_accepted_values(model, column_name, values, quote=True) %}\n {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_not_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0528169}, "macro.dbt_utils.default__test_not_accepted_values": {"unique_id": "macro.dbt_utils.default__test_not_accepted_values", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/not_accepted_values.sql", "original_file_path": "macros/generic_tests/not_accepted_values.sql", "name": "default__test_not_accepted_values", "macro_sql": "{% macro default__test_not_accepted_values(model, column_name, values, quote=True) %}\nwith all_values as (\n\n select distinct\n {{ column_name }} as value_field\n\n from {{ model }}\n\n),\n\nvalidation_errors as (\n\n select\n value_field\n\n from all_values\n where value_field in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n )\n\n)\n\nselect *\nfrom validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.053282}, "macro.dbt_utils.test_unique_where": {"unique_id": "macro.dbt_utils.test_unique_where", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/test_unique_where.sql", "original_file_path": "macros/generic_tests/test_unique_where.sql", "name": "test_unique_where", "macro_sql": "{% test unique_where(model, column_name) %}\r\n {%- set deprecation_warning = '\r\n Warning: `dbt_utils.unique_where` is no longer supported.\r\n Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config.\r\n ' -%}\r\n {%- do exceptions.warn(deprecation_warning) -%}\r\n {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }}\r\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_unique_where"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0538828}, "macro.dbt_utils.default__test_unique_where": {"unique_id": "macro.dbt_utils.default__test_unique_where", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/test_unique_where.sql", "original_file_path": "macros/generic_tests/test_unique_where.sql", "name": "default__test_unique_where", "macro_sql": "{% macro default__test_unique_where(model, column_name) %}\r\n {{ return(test_unique(model, column_name)) }}\r\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.test_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.054086}, "macro.dbt_utils.test_at_least_one": {"unique_id": "macro.dbt_utils.test_at_least_one", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/at_least_one.sql", "original_file_path": "macros/generic_tests/at_least_one.sql", "name": "test_at_least_one", "macro_sql": "{% test at_least_one(model, column_name) %}\n {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_at_least_one"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.054533}, "macro.dbt_utils.default__test_at_least_one": {"unique_id": "macro.dbt_utils.default__test_at_least_one", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/at_least_one.sql", "original_file_path": "macros/generic_tests/at_least_one.sql", "name": "default__test_at_least_one", "macro_sql": "{% macro default__test_at_least_one(model, column_name) %}\n\nselect *\nfrom (\n select\n {# In TSQL, subquery aggregate columns need aliases #}\n {# thus: a filler col name, 'filler_column' #}\n count({{ column_name }}) as filler_column\n\n from {{ model }}\n\n having count({{ column_name }}) = 0\n\n) validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.054726}, "macro.dbt_utils.test_unique_combination_of_columns": {"unique_id": "macro.dbt_utils.test_unique_combination_of_columns", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/unique_combination_of_columns.sql", "original_file_path": "macros/generic_tests/unique_combination_of_columns.sql", "name": "test_unique_combination_of_columns", "macro_sql": "{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %}\n {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_unique_combination_of_columns"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0555038}, "macro.dbt_utils.default__test_unique_combination_of_columns": {"unique_id": "macro.dbt_utils.default__test_unique_combination_of_columns", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/unique_combination_of_columns.sql", "original_file_path": "macros/generic_tests/unique_combination_of_columns.sql", "name": "default__test_unique_combination_of_columns", "macro_sql": "{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %}\n\n{% if not quote_columns %}\n {%- set column_list=combination_of_columns %}\n{% elif quote_columns %}\n {%- set column_list=[] %}\n {% for column in combination_of_columns -%}\n {% set column_list = column_list.append( adapter.quote(column) ) %}\n {%- endfor %}\n{% else %}\n {{ exceptions.raise_compiler_error(\n \"`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '\" ~ quote ~\"'.'\"\n ) }}\n{% endif %}\n\n{%- set columns_csv=column_list | join(', ') %}\n\n\nwith validation_errors as (\n\n select\n {{ columns_csv }}\n from {{ model }}\n group by {{ columns_csv }}\n having count(*) > 1\n\n)\n\nselect *\nfrom validation_errors\n\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.056224}, "macro.dbt_utils.test_cardinality_equality": {"unique_id": "macro.dbt_utils.test_cardinality_equality", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/cardinality_equality.sql", "original_file_path": "macros/generic_tests/cardinality_equality.sql", "name": "test_cardinality_equality", "macro_sql": "{% test cardinality_equality(model, column_name, to, field) %}\n {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_cardinality_equality"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.056888}, "macro.dbt_utils.default__test_cardinality_equality": {"unique_id": "macro.dbt_utils.default__test_cardinality_equality", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/cardinality_equality.sql", "original_file_path": "macros/generic_tests/cardinality_equality.sql", "name": "default__test_cardinality_equality", "macro_sql": "{% macro default__test_cardinality_equality(model, column_name, to, field) %}\n\n{# T-SQL does not let you use numbers as aliases for columns #}\n{# Thus, no \"GROUP BY 1\" #}\n\nwith table_a as (\nselect\n {{ column_name }},\n count(*) as num_rows\nfrom {{ model }}\ngroup by {{ column_name }}\n),\n\ntable_b as (\nselect\n {{ field }},\n count(*) as num_rows\nfrom {{ to }}\ngroup by {{ field }}\n),\n\nexcept_a as (\n select *\n from table_a\n {{ dbt_utils.except() }}\n select *\n from table_b\n),\n\nexcept_b as (\n select *\n from table_b\n {{ dbt_utils.except() }}\n select *\n from table_a\n),\n\nunioned as (\n select *\n from except_a\n union all\n select *\n from except_b\n)\n\nselect *\nfrom unioned\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.except"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0572631}, "macro.dbt_utils.test_expression_is_true": {"unique_id": "macro.dbt_utils.test_expression_is_true", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/expression_is_true.sql", "original_file_path": "macros/generic_tests/expression_is_true.sql", "name": "test_expression_is_true", "macro_sql": "{% test expression_is_true(model, expression, column_name=None, condition='1=1') %}\n{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #}\n{# ref https://stackoverflow.com/a/7170753/3842610 #}\n {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0578518}, "macro.dbt_utils.default__test_expression_is_true": {"unique_id": "macro.dbt_utils.default__test_expression_is_true", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/expression_is_true.sql", "original_file_path": "macros/generic_tests/expression_is_true.sql", "name": "default__test_expression_is_true", "macro_sql": "{% macro default__test_expression_is_true(model, expression, column_name, condition) %}\n\nwith meet_condition as (\n select * from {{ model }} where {{ condition }}\n)\n\nselect\n *\nfrom meet_condition\n{% if column_name is none %}\nwhere not({{ expression }})\n{%- else %}\nwhere not({{ column_name }} {{ expression }})\n{%- endif %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.058173}, "macro.dbt_utils.test_not_null_proportion": {"unique_id": "macro.dbt_utils.test_not_null_proportion", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/not_null_proportion.sql", "original_file_path": "macros/generic_tests/not_null_proportion.sql", "name": "test_not_null_proportion", "macro_sql": "{% macro test_not_null_proportion(model) %}\n {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_not_null_proportion"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0587032}, "macro.dbt_utils.default__test_not_null_proportion": {"unique_id": "macro.dbt_utils.default__test_not_null_proportion", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/not_null_proportion.sql", "original_file_path": "macros/generic_tests/not_null_proportion.sql", "name": "default__test_not_null_proportion", "macro_sql": "{% macro default__test_not_null_proportion(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %}\n{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %}\n\nwith validation as (\n select\n sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion\n from {{ model }}\n),\nvalidation_errors as (\n select\n not_null_proportion\n from validation\n where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }}\n)\nselect\n *\nfrom validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.059274}, "macro.dbt_utils.test_sequential_values": {"unique_id": "macro.dbt_utils.test_sequential_values", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/sequential_values.sql", "original_file_path": "macros/generic_tests/sequential_values.sql", "name": "test_sequential_values", "macro_sql": "{% test sequential_values(model, column_name, interval=1, datepart=None) %}\n\n {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_sequential_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.060053}, "macro.dbt_utils.default__test_sequential_values": {"unique_id": "macro.dbt_utils.default__test_sequential_values", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/sequential_values.sql", "original_file_path": "macros/generic_tests/sequential_values.sql", "name": "default__test_sequential_values", "macro_sql": "{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %}\n\n{% set previous_column_name = \"previous_\" ~ dbt_utils.slugify(column_name) %}\n\nwith windowed as (\n\n select\n {{ column_name }},\n lag({{ column_name }}) over (\n order by {{ column_name }}\n ) as {{ previous_column_name }}\n from {{ model }}\n),\n\nvalidation_errors as (\n select\n *\n from windowed\n {% if datepart %}\n where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }}))\n {% else %}\n where not({{ column_name }} = {{ previous_column_name }} + {{ interval }})\n {% endif %}\n)\n\nselect *\nfrom validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.slugify", "macro.dbt_utils.type_timestamp", "macro.dbt_utils.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.060805}, "macro.dbt_utils.test_not_null_where": {"unique_id": "macro.dbt_utils.test_not_null_where", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/test_not_null_where.sql", "original_file_path": "macros/generic_tests/test_not_null_where.sql", "name": "test_not_null_where", "macro_sql": "{% test not_null_where(model, column_name) %}\r\n {%- set deprecation_warning = '\r\n Warning: `dbt_utils.not_null_where` is no longer supported.\r\n Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config.\r\n ' -%}\r\n {%- do exceptions.warn(deprecation_warning) -%}\r\n {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }}\r\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_not_null_where"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.061328}, "macro.dbt_utils.default__test_not_null_where": {"unique_id": "macro.dbt_utils.default__test_not_null_where", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/test_not_null_where.sql", "original_file_path": "macros/generic_tests/test_not_null_where.sql", "name": "default__test_not_null_where", "macro_sql": "{% macro default__test_not_null_where(model, column_name) %}\r\n {{ return(test_not_null(model, column_name)) }}\r\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.061512}, "macro.dbt_utils.test_equality": {"unique_id": "macro.dbt_utils.test_equality", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/equality.sql", "original_file_path": "macros/generic_tests/equality.sql", "name": "test_equality", "macro_sql": "{% test equality(model, compare_model, compare_columns=None) %}\n {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_equality"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.062356}, "macro.dbt_utils.default__test_equality": {"unique_id": "macro.dbt_utils.default__test_equality", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/equality.sql", "original_file_path": "macros/generic_tests/equality.sql", "name": "default__test_equality", "macro_sql": "{% macro default__test_equality(model, compare_model, compare_columns=None) %}\n\n{% set set_diff %}\n count(*) + coalesce(abs(\n sum(case when which_diff = 'a_minus_b' then 1 else 0 end) -\n sum(case when which_diff = 'b_minus_a' then 1 else 0 end)\n ), 0)\n{% endset %}\n\n{#-- Needs to be set at parse time, before we return '' below --#}\n{{ config(fail_calc = set_diff) }}\n\n{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}\n{%- if not execute -%}\n {{ return('') }}\n{% endif %}\n\n-- setup\n{%- do dbt_utils._is_relation(model, 'test_equality') -%}\n\n{#-\nIf the compare_cols arg is provided, we can run this test without querying the\ninformation schema\u00a0\u2014 this allows the model to be an ephemeral model\n-#}\n\n{%- if not compare_columns -%}\n {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%}\n {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%}\n{%- endif -%}\n\n{% set compare_cols_csv = compare_columns | join(', ') %}\n\nwith a as (\n\n select * from {{ model }}\n\n),\n\nb as (\n\n select * from {{ compare_model }}\n\n),\n\na_minus_b as (\n\n select {{compare_cols_csv}} from a\n {{ dbt_utils.except() }}\n select {{compare_cols_csv}} from b\n\n),\n\nb_minus_a as (\n\n select {{compare_cols_csv}} from b\n {{ dbt_utils.except() }}\n select {{compare_cols_csv}} from a\n\n),\n\nunioned as (\n\n select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b\n union all\n select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a\n\n)\n\nselect * from unioned\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_relation", "macro.dbt_utils._is_ephemeral", "macro.dbt_utils.except"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0633469}, "macro.dbt_utils.test_mutually_exclusive_ranges": {"unique_id": "macro.dbt_utils.test_mutually_exclusive_ranges", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/mutually_exclusive_ranges.sql", "original_file_path": "macros/generic_tests/mutually_exclusive_ranges.sql", "name": "test_mutually_exclusive_ranges", "macro_sql": "{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %}\n {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__test_mutually_exclusive_ranges"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.066734}, "macro.dbt_utils.default__test_mutually_exclusive_ranges": {"unique_id": "macro.dbt_utils.default__test_mutually_exclusive_ranges", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/generic_tests/mutually_exclusive_ranges.sql", "original_file_path": "macros/generic_tests/mutually_exclusive_ranges.sql", "name": "default__test_mutually_exclusive_ranges", "macro_sql": "{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %}\n{% if gaps == 'not_allowed' %}\n {% set allow_gaps_operator='=' %}\n {% set allow_gaps_operator_in_words='equal_to' %}\n{% elif gaps == 'allowed' %}\n {% set allow_gaps_operator='<=' %}\n {% set allow_gaps_operator_in_words='less_than_or_equal_to' %}\n{% elif gaps == 'required' %}\n {% set allow_gaps_operator='<' %}\n {% set allow_gaps_operator_in_words='less_than' %}\n{% else %}\n {{ exceptions.raise_compiler_error(\n \"`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '\" ~ gaps ~\"'.'\"\n ) }}\n{% endif %}\n{% if not zero_length_range_allowed %}\n {% set allow_zero_length_operator='<' %}\n {% set allow_zero_length_operator_in_words='less_than' %}\n{% elif zero_length_range_allowed %}\n {% set allow_zero_length_operator='<=' %}\n {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %}\n{% else %}\n {{ exceptions.raise_compiler_error(\n \"`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '\" ~ zero_length_range_allowed ~\"'.'\"\n ) }}\n{% endif %}\n\n{% set partition_clause=\"partition by \" ~ partition_by if partition_by else '' %}\n\nwith window_functions as (\n\n select\n {% if partition_by %}\n {{ partition_by }} as partition_by_col,\n {% endif %}\n {{ lower_bound_column }} as lower_bound,\n {{ upper_bound_column }} as upper_bound,\n\n lead({{ lower_bound_column }}) over (\n {{ partition_clause }}\n order by {{ lower_bound_column }}\n ) as next_lower_bound,\n\n row_number() over (\n {{ partition_clause }}\n order by {{ lower_bound_column }} desc\n ) = 1 as is_last_record\n\n from {{ model }}\n\n),\n\ncalc as (\n -- We want to return records where one of our assumptions fails, so we'll use\n -- the `not` function with `and` statements so we can write our assumptions nore cleanly\n select\n *,\n\n -- For each record: lower_bound should be < upper_bound.\n -- Coalesce it to return an error on the null case (implicit assumption\n -- these columns are not_null)\n coalesce(\n lower_bound {{ allow_zero_length_operator }} upper_bound,\n false\n ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound,\n\n -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound.\n -- Coalesce it to handle null cases for the last record.\n coalesce(\n upper_bound {{ allow_gaps_operator }} next_lower_bound,\n is_last_record,\n false\n ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound\n\n from window_functions\n\n),\n\nvalidation_errors as (\n\n select\n *\n from calc\n\n where not(\n -- THE FOLLOWING SHOULD BE TRUE --\n lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound\n and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound\n )\n)\n\nselect * from validation_errors\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.068387}, "macro.dbt_utils.pretty_log_format": {"unique_id": "macro.dbt_utils.pretty_log_format", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/pretty_log_format.sql", "original_file_path": "macros/jinja_helpers/pretty_log_format.sql", "name": "pretty_log_format", "macro_sql": "{% macro pretty_log_format(message) %}\n {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__pretty_log_format"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0687711}, "macro.dbt_utils.default__pretty_log_format": {"unique_id": "macro.dbt_utils.default__pretty_log_format", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/pretty_log_format.sql", "original_file_path": "macros/jinja_helpers/pretty_log_format.sql", "name": "default__pretty_log_format", "macro_sql": "{% macro default__pretty_log_format(message) %}\n {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.pretty_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.068948}, "macro.dbt_utils.pretty_time": {"unique_id": "macro.dbt_utils.pretty_time", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/pretty_time.sql", "original_file_path": "macros/jinja_helpers/pretty_time.sql", "name": "pretty_time", "macro_sql": "{% macro pretty_time(format='%H:%M:%S') %}\n {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__pretty_time"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0693262}, "macro.dbt_utils.default__pretty_time": {"unique_id": "macro.dbt_utils.default__pretty_time", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/pretty_time.sql", "original_file_path": "macros/jinja_helpers/pretty_time.sql", "name": "default__pretty_time", "macro_sql": "{% macro default__pretty_time(format='%H:%M:%S') %}\n {{ return(modules.datetime.datetime.now().strftime(format)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.069536}, "macro.dbt_utils.log_info": {"unique_id": "macro.dbt_utils.log_info", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/log_info.sql", "original_file_path": "macros/jinja_helpers/log_info.sql", "name": "log_info", "macro_sql": "{% macro log_info(message) %}\n {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__log_info"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.06989}, "macro.dbt_utils.default__log_info": {"unique_id": "macro.dbt_utils.default__log_info", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/log_info.sql", "original_file_path": "macros/jinja_helpers/log_info.sql", "name": "default__log_info", "macro_sql": "{% macro default__log_info(message) %}\n {{ log(dbt_utils.pretty_log_format(message), info=True) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.pretty_log_format"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.070074}, "macro.dbt_utils.slugify": {"unique_id": "macro.dbt_utils.slugify", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/jinja_helpers/slugify.sql", "original_file_path": "macros/jinja_helpers/slugify.sql", "name": "slugify", "macro_sql": "{% macro slugify(string) %}\n\n{#- Lower case the string -#}\n{% set string = string | lower %}\n{#- Replace spaces and dashes with underscores -#}\n{% set string = modules.re.sub('[ -]+', '_', string) %}\n{#- Only take letters, numbers, and underscores -#}\n{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %}\n\n{{ return(string) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.070663}, "macro.dbt_utils.get_intervals_between": {"unique_id": "macro.dbt_utils.get_intervals_between", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/date_spine.sql", "original_file_path": "macros/sql/date_spine.sql", "name": "get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0714738}, "macro.dbt_utils.default__get_intervals_between": {"unique_id": "macro.dbt_utils.default__get_intervals_between", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/date_spine.sql", "original_file_path": "macros/sql/date_spine.sql", "name": "default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{dbt_utils.datediff(start_date, end_date, datepart)}}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_utils.datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.072143}, "macro.dbt_utils.date_spine": {"unique_id": "macro.dbt_utils.date_spine", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/date_spine.sql", "original_file_path": "macros/sql/date_spine.sql", "name": "date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.072397}, "macro.dbt_utils.default__date_spine": {"unique_id": "macro.dbt_utils.default__date_spine", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/date_spine.sql", "original_file_path": "macros/sql/date_spine.sql", "name": "default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n{# call as follows:\n\ndate_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dateadd(week, 1, current_date)\"\n) #}\n\n\nwith rawdata as (\n\n {{dbt_utils.generate_series(\n dbt_utils.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n),\n\nall_periods as (\n\n select (\n {{\n dbt_utils.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n),\n\nfiltered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n)\n\nselect * from filtered\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.generate_series", "macro.dbt_utils.get_intervals_between", "macro.dbt_utils.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.072799}, "macro.dbt_utils.nullcheck_table": {"unique_id": "macro.dbt_utils.nullcheck_table", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/nullcheck_table.sql", "original_file_path": "macros/sql/nullcheck_table.sql", "name": "nullcheck_table", "macro_sql": "{% macro nullcheck_table(relation) %}\n {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__nullcheck_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.073217}, "macro.dbt_utils.default__nullcheck_table": {"unique_id": "macro.dbt_utils.default__nullcheck_table", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/nullcheck_table.sql", "original_file_path": "macros/sql/nullcheck_table.sql", "name": "default__nullcheck_table", "macro_sql": "{% macro default__nullcheck_table(relation) %}\n\n {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%}\n {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%}\n {% set cols = adapter.get_columns_in_relation(relation) %}\n\n select {{ dbt_utils.nullcheck(cols) }}\n from {{relation}}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_relation", "macro.dbt_utils._is_ephemeral", "macro.dbt_utils.nullcheck"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0736582}, "macro.dbt_utils.get_relations_by_pattern": {"unique_id": "macro.dbt_utils.get_relations_by_pattern", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_relations_by_pattern.sql", "original_file_path": "macros/sql/get_relations_by_pattern.sql", "name": "get_relations_by_pattern", "macro_sql": "{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %}\n {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_relations_by_pattern"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.074462}, "macro.dbt_utils.default__get_relations_by_pattern": {"unique_id": "macro.dbt_utils.default__get_relations_by_pattern", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_relations_by_pattern.sql", "original_file_path": "macros/sql/get_relations_by_pattern.sql", "name": "default__get_relations_by_pattern", "macro_sql": "{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %}\n\n {%- call statement('get_tables', fetch_result=True) %}\n\n {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }}\n\n {%- endcall -%}\n\n {%- set table_list = load_result('get_tables') -%}\n\n {%- if table_list and table_list['table'] -%}\n {%- set tbl_relations = [] -%}\n {%- for row in table_list['table'] -%}\n {%- set tbl_relation = api.Relation.create(\n database=database,\n schema=row.table_schema,\n identifier=row.table_name,\n type=row.table_type\n ) -%}\n {%- do tbl_relations.append(tbl_relation) -%}\n {%- endfor -%}\n\n {{ return(tbl_relations) }}\n {%- else -%}\n {{ return([]) }}\n {%- endif -%}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_utils.get_tables_by_pattern_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.075407}, "macro.dbt_utils.get_powers_of_two": {"unique_id": "macro.dbt_utils.get_powers_of_two", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/generate_series.sql", "original_file_path": "macros/sql/generate_series.sql", "name": "get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0763578}, "macro.dbt_utils.default__get_powers_of_two": {"unique_id": "macro.dbt_utils.default__get_powers_of_two", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/generate_series.sql", "original_file_path": "macros/sql/generate_series.sql", "name": "default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0768168}, "macro.dbt_utils.generate_series": {"unique_id": "macro.dbt_utils.generate_series", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/generate_series.sql", "original_file_path": "macros/sql/generate_series.sql", "name": "generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0770261}, "macro.dbt_utils.default__generate_series": {"unique_id": "macro.dbt_utils.default__generate_series", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/generate_series.sql", "original_file_path": "macros/sql/generate_series.sql", "name": "default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt_utils.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.077594}, "macro.dbt_utils.get_relations_by_prefix": {"unique_id": "macro.dbt_utils.get_relations_by_prefix", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_relations_by_prefix.sql", "original_file_path": "macros/sql/get_relations_by_prefix.sql", "name": "get_relations_by_prefix", "macro_sql": "{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %}\n {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_relations_by_prefix"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.078399}, "macro.dbt_utils.default__get_relations_by_prefix": {"unique_id": "macro.dbt_utils.default__get_relations_by_prefix", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_relations_by_prefix.sql", "original_file_path": "macros/sql/get_relations_by_prefix.sql", "name": "default__get_relations_by_prefix", "macro_sql": "{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %}\n\n {%- call statement('get_tables', fetch_result=True) %}\n\n {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }}\n\n {%- endcall -%}\n\n {%- set table_list = load_result('get_tables') -%}\n\n {%- if table_list and table_list['table'] -%}\n {%- set tbl_relations = [] -%}\n {%- for row in table_list['table'] -%}\n {%- set tbl_relation = api.Relation.create(\n database=database,\n schema=row.table_schema,\n identifier=row.table_name,\n type=row.table_type\n ) -%}\n {%- do tbl_relations.append(tbl_relation) -%}\n {%- endfor -%}\n\n {{ return(tbl_relations) }}\n {%- else -%}\n {{ return([]) }}\n {%- endif -%}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_utils.get_tables_by_prefix_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.079337}, "macro.dbt_utils.get_tables_by_prefix_sql": {"unique_id": "macro.dbt_utils.get_tables_by_prefix_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_tables_by_prefix_sql.sql", "original_file_path": "macros/sql/get_tables_by_prefix_sql.sql", "name": "get_tables_by_prefix_sql", "macro_sql": "{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %}\n {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_tables_by_prefix_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0798552}, "macro.dbt_utils.default__get_tables_by_prefix_sql": {"unique_id": "macro.dbt_utils.default__get_tables_by_prefix_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_tables_by_prefix_sql.sql", "original_file_path": "macros/sql/get_tables_by_prefix_sql.sql", "name": "default__get_tables_by_prefix_sql", "macro_sql": "{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %}\n\n {{ dbt_utils.get_tables_by_pattern_sql(\n schema_pattern = schema,\n table_pattern = prefix ~ '%',\n exclude = exclude,\n database = database\n ) }}\n \n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.get_tables_by_pattern_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.080168}, "macro.dbt_utils.star": {"unique_id": "macro.dbt_utils.star", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/star.sql", "original_file_path": "macros/sql/star.sql", "name": "star", "macro_sql": "{% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%}\n {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__star"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.080898}, "macro.dbt_utils.default__star": {"unique_id": "macro.dbt_utils.default__star", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/star.sql", "original_file_path": "macros/sql/star.sql", "name": "default__star", "macro_sql": "{% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%}\n {%- do dbt_utils._is_relation(from, 'star') -%}\n {%- do dbt_utils._is_ephemeral(from, 'star') -%}\n\n {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}\n {%- if not execute -%}\n {{ return('*') }}\n {% endif %}\n\n {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %}\n\n {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%}\n {%- if not loop.last %},{{ '\\n ' }}{% endif %}\n\n {%- endfor -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_relation", "macro.dbt_utils._is_ephemeral", "macro.dbt_utils.get_filtered_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0818272}, "macro.dbt_utils.unpivot": {"unique_id": "macro.dbt_utils.unpivot", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/unpivot.sql", "original_file_path": "macros/sql/unpivot.sql", "name": "unpivot", "macro_sql": "{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%}\n {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__unpivot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.083801}, "macro.dbt_utils.default__unpivot": {"unique_id": "macro.dbt_utils.default__unpivot", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/unpivot.sql", "original_file_path": "macros/sql/unpivot.sql", "name": "default__unpivot", "macro_sql": "{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%}\n\n {% if table %}\n {%- set error_message = '\n Warning: the `unpivot` macro no longer accepts a `table` parameter. \\\n This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \\\n The {}.{} model triggered this warning. \\\n '.format(model.package_name, model.name) -%}\n {%- do exceptions.warn(error_message) -%}\n {% endif %}\n\n {% if relation and table %}\n {{ exceptions.raise_compiler_error(\"Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`).\") }}\n {% elif not relation and table %}\n {% set relation=table %}\n {% elif not relation and not table %}\n {{ exceptions.raise_compiler_error(\"Error: argument `relation` is required for `unpivot` macro.\") }}\n {% endif %}\n\n {%- set exclude = exclude if exclude is not none else [] %}\n {%- set remove = remove if remove is not none else [] %}\n\n {%- set include_cols = [] %}\n\n {%- set table_columns = {} %}\n\n {%- do table_columns.update({relation: []}) %}\n\n {%- do dbt_utils._is_relation(relation, 'unpivot') -%}\n {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%}\n {%- set cols = adapter.get_columns_in_relation(relation) %}\n\n {%- for col in cols -%}\n {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%}\n {% do include_cols.append(col) %}\n {%- endif %}\n {%- endfor %}\n\n\n {%- for col in include_cols -%}\n select\n {%- for exclude_col in exclude %}\n {{ exclude_col }},\n {%- endfor %}\n\n cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }},\n cast( {% if col.data_type == 'boolean' %}\n {{ dbt_utils.cast_bool_to_text(col.column) }}\n {% else %}\n {{ col.column }}\n {% endif %}\n as {{ cast_to }}) as {{ value_name }}\n\n from {{ relation }}\n\n {% if not loop.last -%}\n union all\n {% endif -%}\n {%- endfor -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_relation", "macro.dbt_utils._is_ephemeral", "macro.dbt_utils.type_string", "macro.dbt_utils.cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0861802}, "macro.dbt_utils.union_relations": {"unique_id": "macro.dbt_utils.union_relations", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/union.sql", "original_file_path": "macros/sql/union.sql", "name": "union_relations", "macro_sql": "{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}\n {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name, where)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__union_relations"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.08895}, "macro.dbt_utils.default__union_relations": {"unique_id": "macro.dbt_utils.default__union_relations", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/union.sql", "original_file_path": "macros/sql/union.sql", "name": "default__union_relations", "macro_sql": "\n\n{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}\n\n {%- if exclude and include -%}\n {{ exceptions.raise_compiler_error(\"Both an exclude and include list were provided to the `union` macro. Only one is allowed\") }}\n {%- endif -%}\n\n {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#}\n {%- if not execute %}\n {{ return('') }}\n {% endif -%}\n\n {%- set column_override = column_override if column_override is not none else {} -%}\n\n {%- set relation_columns = {} -%}\n {%- set column_superset = {} -%}\n\n {%- for relation in relations -%}\n\n {%- do relation_columns.update({relation: []}) -%}\n\n {%- do dbt_utils._is_relation(relation, 'union_relations') -%}\n {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%}\n {%- set cols = adapter.get_columns_in_relation(relation) -%}\n {%- for col in cols -%}\n\n {#- If an exclude list was provided and the column is in the list, do nothing -#}\n {%- if exclude and col.column in exclude -%}\n\n {#- If an include list was provided and the column is not in the list, do nothing -#}\n {%- elif include and col.column not in include -%}\n\n {#- Otherwise add the column to the column superset -#}\n {%- else -%}\n\n {#- update the list of columns in this relation -#}\n {%- do relation_columns[relation].append(col.column) -%}\n\n {%- if col.column in column_superset -%}\n\n {%- set stored = column_superset[col.column] -%}\n {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%}\n\n {%- do column_superset.update({col.column: col}) -%}\n\n {%- endif %}\n\n {%- else -%}\n\n {%- do column_superset.update({col.column: col}) -%}\n\n {%- endif -%}\n\n {%- endif -%}\n\n {%- endfor -%}\n {%- endfor -%}\n\n {%- set ordered_column_names = column_superset.keys() -%}\n\n {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %}\n {%- set relations_string -%}\n {%- for relation in relations -%}\n {{ relation.name }}\n {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n {%- endset -%}\n\n {%- set error_message -%}\n There were no columns found to union for relations {{ relations_string }}\n {%- endset -%}\n\n {{ exceptions.raise_compiler_error(error_message) }}\n {%- endif -%}\n\n {%- for relation in relations %}\n\n (\n select\n\n cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }},\n {% for col_name in ordered_column_names -%}\n\n {%- set col = column_superset[col_name] %}\n {%- set col_type = column_override.get(col.column, col.data_type) %}\n {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %}\n cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%}\n\n {%- endfor %}\n\n from {{ relation }}\n\n {% if where -%}\n where {{ where }}\n {%- endif %}\n )\n\n {% if not loop.last -%}\n union all\n {% endif -%}\n\n {%- endfor -%}\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_relation", "macro.dbt_utils._is_ephemeral", "macro.dbt_utils.string_literal", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.092146}, "macro.dbt_utils.group_by": {"unique_id": "macro.dbt_utils.group_by", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/groupby.sql", "original_file_path": "macros/sql/groupby.sql", "name": "group_by", "macro_sql": "{%- macro group_by(n) -%}\n {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__group_by"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.092552}, "macro.dbt_utils.default__group_by": {"unique_id": "macro.dbt_utils.default__group_by", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/groupby.sql", "original_file_path": "macros/sql/groupby.sql", "name": "default__group_by", "macro_sql": "\n\n{%- macro default__group_by(n) -%}\n\n group by {% for i in range(1, n + 1) -%}\n {{ i }}{{ ',' if not loop.last }} \n {%- endfor -%}\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.09282}, "macro.dbt_utils.deduplicate": {"unique_id": "macro.dbt_utils.deduplicate", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/deduplicate.sql", "original_file_path": "macros/sql/deduplicate.sql", "name": "deduplicate", "macro_sql": "{%- macro deduplicate(relation, partition_by, order_by=none, relation_alias=none) -%}\n\n {%- set error_message_group_by -%}\nWarning: the `group_by` parameter of the `deduplicate` macro is no longer supported and will be deprecated in a future release of dbt-utils.\nUse `partition_by` instead.\nThe {{ model.package_name }}.{{ model.name }} model triggered this warning.\n {%- endset -%}\n\n {% if kwargs.get('group_by') %}\n {%- do exceptions.warn(error_message_group_by) -%}\n {%- endif -%}\n\n {%- set error_message_order_by -%}\nWarning: `order_by` as an optional parameter of the `deduplicate` macro is no longer supported and will be deprecated in a future release of dbt-utils.\nSupply a non-null value for `order_by` instead.\nThe {{ model.package_name }}.{{ model.name }} model triggered this warning.\n {%- endset -%}\n\n {% if not order_by %}\n {%- do exceptions.warn(error_message_order_by) -%}\n {%- endif -%}\n\n {%- set error_message_alias -%}\nWarning: the `relation_alias` parameter of the `deduplicate` macro is no longer supported and will be deprecated in a future release of dbt-utils.\nIf you were using `relation_alias` to point to a CTE previously then you can now pass the alias directly to `relation` instead.\nThe {{ model.package_name }}.{{ model.name }} model triggered this warning.\n {%- endset -%}\n\n {% if relation_alias %}\n {%- do exceptions.warn(error_message_alias) -%}\n {%- endif -%}\n\n {% set partition_by = partition_by or kwargs.get('group_by') %}\n {% set relation = relation_alias or relation %}\n {% set order_by = order_by or \"'1'\" %}\n\n {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, partition_by, order_by)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__deduplicate"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0955222}, "macro.dbt_utils.default__deduplicate": {"unique_id": "macro.dbt_utils.default__deduplicate", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/deduplicate.sql", "original_file_path": "macros/sql/deduplicate.sql", "name": "default__deduplicate", "macro_sql": "\n\n{%- macro default__deduplicate(relation, partition_by, order_by) -%}\n\n with row_numbered as (\n select\n _inner.*,\n row_number() over (\n partition by {{ partition_by }}\n order by {{ order_by }}\n ) as rn\n from {{ relation }} as _inner\n )\n\n select\n distinct data.*\n from {{ relation }} as data\n {#\n -- Not all DBs will support natural joins but the ones that do include:\n -- Oracle, MySQL, SQLite, Redshift, Teradata, Materialize, Databricks\n -- Apache Spark, SingleStore, Vertica\n -- Those that do not appear to support natural joins include:\n -- SQLServer, Trino, Presto, Rockset, Athena\n #}\n natural join row_numbered\n where row_numbered.rn = 1\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.095763}, "macro.dbt_utils.redshift__deduplicate": {"unique_id": "macro.dbt_utils.redshift__deduplicate", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/deduplicate.sql", "original_file_path": "macros/sql/deduplicate.sql", "name": "redshift__deduplicate", "macro_sql": "{% macro redshift__deduplicate(relation, partition_by, order_by) -%}\n\n {{ return(dbt_utils.default__deduplicate(relation, partition_by, order_by=order_by)) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__deduplicate"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0959811}, "macro.dbt_utils.postgres__deduplicate": {"unique_id": "macro.dbt_utils.postgres__deduplicate", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/deduplicate.sql", "original_file_path": "macros/sql/deduplicate.sql", "name": "postgres__deduplicate", "macro_sql": "\n{%- macro postgres__deduplicate(relation, partition_by, order_by) -%}\n\n select\n distinct on ({{ partition_by }}) *\n from {{ relation }}\n order by {{ partition_by }}{{ ',' ~ order_by }}\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.096192}, "macro.dbt_utils.snowflake__deduplicate": {"unique_id": "macro.dbt_utils.snowflake__deduplicate", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/deduplicate.sql", "original_file_path": "macros/sql/deduplicate.sql", "name": "snowflake__deduplicate", "macro_sql": "\n{%- macro snowflake__deduplicate(relation, partition_by, order_by) -%}\n\n select *\n from {{ relation }}\n qualify\n row_number() over (\n partition by {{ partition_by }}\n order by {{ order_by }}\n ) = 1\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0963688}, "macro.dbt_utils.bigquery__deduplicate": {"unique_id": "macro.dbt_utils.bigquery__deduplicate", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/deduplicate.sql", "original_file_path": "macros/sql/deduplicate.sql", "name": "bigquery__deduplicate", "macro_sql": "\n{%- macro bigquery__deduplicate(relation, partition_by, order_by) -%}\n\n select unique.*\n from (\n select\n array_agg (\n original\n order by {{ order_by }}\n limit 1\n )[offset(0)] unique\n from {{ relation }} original\n group by {{ partition_by }}\n )\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0965528}, "macro.dbt_utils.surrogate_key": {"unique_id": "macro.dbt_utils.surrogate_key", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/surrogate_key.sql", "original_file_path": "macros/sql/surrogate_key.sql", "name": "surrogate_key", "macro_sql": "{%- macro surrogate_key(field_list) -%}\n {# needed for safe_add to allow for non-keyword arguments see SO post #}\n {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #}\n {% set frustrating_jinja_feature = varargs %}\n {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__surrogate_key"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0972538}, "macro.dbt_utils.default__surrogate_key": {"unique_id": "macro.dbt_utils.default__surrogate_key", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/surrogate_key.sql", "original_file_path": "macros/sql/surrogate_key.sql", "name": "default__surrogate_key", "macro_sql": "\n\n{%- macro default__surrogate_key(field_list) -%}\n\n{%- if varargs|length >= 1 or field_list is string %}\n\n{%- set error_message = '\nWarning: the `surrogate_key` macro now takes a single list argument instead of \\\nmultiple string arguments. Support for multiple string arguments will be \\\ndeprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \\\n'.format(model.package_name, model.name) -%}\n\n{%- do exceptions.warn(error_message) -%}\n\n{# first argument is not included in varargs, so add first element to field_list_xf #}\n{%- set field_list_xf = [field_list] -%}\n\n{%- for field in varargs %}\n{%- set _ = field_list_xf.append(field) -%}\n{%- endfor -%}\n\n{%- else -%}\n\n{# if using list, just set field_list_xf as field_list #}\n{%- set field_list_xf = field_list -%}\n\n{%- endif -%}\n\n\n{%- set fields = [] -%}\n\n{%- for field in field_list_xf -%}\n\n {%- set _ = fields.append(\n \"coalesce(cast(\" ~ field ~ \" as \" ~ dbt_utils.type_string() ~ \"), '')\"\n ) -%}\n\n {%- if not loop.last %}\n {%- set _ = fields.append(\"'-'\") -%}\n {%- endif -%}\n\n{%- endfor -%}\n\n{{dbt_utils.hash(dbt_utils.concat(fields))}}\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_string", "macro.dbt_utils.hash", "macro.dbt_utils.concat"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.09828}, "macro.dbt_utils.safe_add": {"unique_id": "macro.dbt_utils.safe_add", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/safe_add.sql", "original_file_path": "macros/sql/safe_add.sql", "name": "safe_add", "macro_sql": "{%- macro safe_add() -%}\n {# needed for safe_add to allow for non-keyword arguments see SO post #}\n {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #}\n {% set frustrating_jinja_feature = varargs %}\n {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__safe_add"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.098772}, "macro.dbt_utils.default__safe_add": {"unique_id": "macro.dbt_utils.default__safe_add", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/safe_add.sql", "original_file_path": "macros/sql/safe_add.sql", "name": "default__safe_add", "macro_sql": "\n\n{%- macro default__safe_add() -%}\n\n{% set fields = [] %}\n\n{%- for field in varargs -%}\n\n {% do fields.append(\"coalesce(\" ~ field ~ \", 0)\") %}\n\n{%- endfor -%}\n\n{{ fields|join(' +\\n ') }}\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.0990741}, "macro.dbt_utils.nullcheck": {"unique_id": "macro.dbt_utils.nullcheck", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/nullcheck.sql", "original_file_path": "macros/sql/nullcheck.sql", "name": "nullcheck", "macro_sql": "{% macro nullcheck(cols) %}\n {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__nullcheck"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.099518}, "macro.dbt_utils.default__nullcheck": {"unique_id": "macro.dbt_utils.default__nullcheck", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/nullcheck.sql", "original_file_path": "macros/sql/nullcheck.sql", "name": "default__nullcheck", "macro_sql": "{% macro default__nullcheck(cols) %}\n{%- for col in cols %}\n\n {% if col.is_string() -%}\n\n nullif({{col.name}},'') as {{col.name}}\n\n {%- else -%}\n\n {{col.name}}\n\n {%- endif -%}\n\n{%- if not loop.last -%} , {%- endif -%}\n\n{%- endfor -%}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.099882}, "macro.dbt_utils.get_tables_by_pattern_sql": {"unique_id": "macro.dbt_utils.get_tables_by_pattern_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_tables_by_pattern_sql.sql", "original_file_path": "macros/sql/get_tables_by_pattern_sql.sql", "name": "get_tables_by_pattern_sql", "macro_sql": "{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}\n {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils')\n (schema_pattern, table_pattern, exclude, database)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__get_tables_by_pattern_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1015968}, "macro.dbt_utils.default__get_tables_by_pattern_sql": {"unique_id": "macro.dbt_utils.default__get_tables_by_pattern_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_tables_by_pattern_sql.sql", "original_file_path": "macros/sql/get_tables_by_pattern_sql.sql", "name": "default__get_tables_by_pattern_sql", "macro_sql": "{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}\n\n select distinct\n table_schema as \"table_schema\",\n table_name as \"table_name\",\n {{ dbt_utils.get_table_types_sql() }}\n from {{ database }}.information_schema.tables\n where table_schema ilike '{{ schema_pattern }}'\n and table_name ilike '{{ table_pattern }}'\n and table_name not ilike '{{ exclude }}'\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.get_table_types_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.101897}, "macro.dbt_utils.bigquery__get_tables_by_pattern_sql": {"unique_id": "macro.dbt_utils.bigquery__get_tables_by_pattern_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_tables_by_pattern_sql.sql", "original_file_path": "macros/sql/get_tables_by_pattern_sql.sql", "name": "bigquery__get_tables_by_pattern_sql", "macro_sql": "{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}\n\n {% if '%' in schema_pattern %}\n {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %}\n {% else %}\n {% set schemata=[schema_pattern] %}\n {% endif %}\n\n {% set sql %}\n {% for schema in schemata %}\n select distinct\n table_schema,\n table_name,\n {{ dbt_utils.get_table_types_sql() }}\n\n from {{ adapter.quote(database) }}.{{ schema }}.INFORMATION_SCHEMA.TABLES\n where lower(table_name) like lower ('{{ table_pattern }}')\n and lower(table_name) not like lower ('{{ exclude }}')\n\n {% if not loop.last %} union all {% endif %}\n\n {% endfor %}\n {% endset %}\n\n {{ return(sql) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._bigquery__get_matching_schemata", "macro.dbt_utils.get_table_types_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1027598}, "macro.dbt_utils._bigquery__get_matching_schemata": {"unique_id": "macro.dbt_utils._bigquery__get_matching_schemata", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_tables_by_pattern_sql.sql", "original_file_path": "macros/sql/get_tables_by_pattern_sql.sql", "name": "_bigquery__get_matching_schemata", "macro_sql": "{% macro _bigquery__get_matching_schemata(schema_pattern, database) %}\n {% if execute %}\n\n {% set sql %}\n select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA\n where lower(schema_name) like lower('{{ schema_pattern }}')\n {% endset %}\n\n {% set results=run_query(sql) %}\n\n {% set schemata=results.columns['schema_name'].values() %}\n\n {{ return(schemata) }}\n\n {% else %}\n\n {{ return([]) }}\n\n {% endif %}\n\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.103317}, "macro.dbt_utils.get_column_values": {"unique_id": "macro.dbt_utils.get_column_values", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_column_values.sql", "original_file_path": "macros/sql/get_column_values.sql", "name": "get_column_values", "macro_sql": "{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none, where=none) -%}\n {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default, where)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_column_values"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.104587}, "macro.dbt_utils.default__get_column_values": {"unique_id": "macro.dbt_utils.default__get_column_values", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_column_values.sql", "original_file_path": "macros/sql/get_column_values.sql", "name": "default__get_column_values", "macro_sql": "{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none, where=none) -%}\n {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}\n {%- if not execute -%}\n {% set default = [] if not default %}\n {{ return(default) }}\n {% endif %}\n\n {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%}\n\n {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #}\n {# TODO: Change the method signature in a future 0.x.0 release #}\n {%- set target_relation = table -%}\n\n {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #}\n {% set relation_exists = (load_relation(target_relation)) is not none %}\n\n {%- call statement('get_column_values', fetch_result=true) %}\n\n {%- if not relation_exists and default is none -%}\n\n {{ exceptions.raise_compiler_error(\"In get_column_values(): relation \" ~ target_relation ~ \" does not exist and no default value was provided.\") }}\n\n {%- elif not relation_exists and default is not none -%}\n\n {{ log(\"Relation \" ~ target_relation ~ \" does not exist. Returning the default value: \" ~ default) }}\n\n {{ return(default) }}\n\n {%- else -%}\n\n\n select\n {{ column }} as value\n\n from {{ target_relation }}\n\n {% if where is not none %}\n where {{ where }}\n {% endif %}\n\n group by {{ column }}\n order by {{ order_by }}\n\n {% if max_records is not none %}\n limit {{ max_records }}\n {% endif %}\n\n {% endif %}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_column_values') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values) }}\n {%- else -%}\n {{ return(default) }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_ephemeral", "macro.dbt.load_relation", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.106267}, "macro.dbt_utils.pivot": {"unique_id": "macro.dbt_utils.pivot", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/pivot.sql", "original_file_path": "macros/sql/pivot.sql", "name": "pivot", "macro_sql": "{% macro pivot(column,\n values,\n alias=True,\n agg='sum',\n cmp='=',\n prefix='',\n suffix='',\n then_value=1,\n else_value=0,\n quote_identifiers=True,\n distinct=False) %}\n {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__pivot"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1074839}, "macro.dbt_utils.default__pivot": {"unique_id": "macro.dbt_utils.default__pivot", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/pivot.sql", "original_file_path": "macros/sql/pivot.sql", "name": "default__pivot", "macro_sql": "{% macro default__pivot(column,\n values,\n alias=True,\n agg='sum',\n cmp='=',\n prefix='',\n suffix='',\n then_value=1,\n else_value=0,\n quote_identifiers=True,\n distinct=False) %}\n {% for value in values %}\n {{ agg }}(\n {% if distinct %} distinct {% endif %}\n case\n when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}'\n then {{ then_value }}\n else {{ else_value }}\n end\n )\n {% if alias %}\n {% if quote_identifiers %}\n as {{ adapter.quote(prefix ~ value ~ suffix) }}\n {% else %}\n as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }}\n {% endif %}\n {% endif %}\n {% if not loop.last %},{% endif %}\n {% endfor %}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.escape_single_quotes", "macro.dbt_utils.slugify"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.108427}, "macro.dbt_utils.get_filtered_columns_in_relation": {"unique_id": "macro.dbt_utils.get_filtered_columns_in_relation", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_filtered_columns_in_relation.sql", "original_file_path": "macros/sql/get_filtered_columns_in_relation.sql", "name": "get_filtered_columns_in_relation", "macro_sql": "{% macro get_filtered_columns_in_relation(from, except=[]) -%}\n {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_filtered_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1090088}, "macro.dbt_utils.default__get_filtered_columns_in_relation": {"unique_id": "macro.dbt_utils.default__get_filtered_columns_in_relation", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_filtered_columns_in_relation.sql", "original_file_path": "macros/sql/get_filtered_columns_in_relation.sql", "name": "default__get_filtered_columns_in_relation", "macro_sql": "{% macro default__get_filtered_columns_in_relation(from, except=[]) -%}\n {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%}\n {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%}\n\n {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}\n {%- if not execute -%}\n {{ return('') }}\n {% endif %}\n\n {%- set include_cols = [] %}\n {%- set cols = adapter.get_columns_in_relation(from) -%}\n {%- set except = except | map(\"lower\") | list %}\n {%- for col in cols -%}\n {%- if col.column|lower not in except -%}\n {% do include_cols.append(col.column) %}\n {%- endif %}\n {%- endfor %}\n\n {{ return(include_cols) }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils._is_relation", "macro.dbt_utils._is_ephemeral"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.109814}, "macro.dbt_utils.get_query_results_as_dict": {"unique_id": "macro.dbt_utils.get_query_results_as_dict", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_query_results_as_dict.sql", "original_file_path": "macros/sql/get_query_results_as_dict.sql", "name": "get_query_results_as_dict", "macro_sql": "{% macro get_query_results_as_dict(query) %}\n {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.default__get_query_results_as_dict"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.110314}, "macro.dbt_utils.default__get_query_results_as_dict": {"unique_id": "macro.dbt_utils.default__get_query_results_as_dict", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_query_results_as_dict.sql", "original_file_path": "macros/sql/get_query_results_as_dict.sql", "name": "default__get_query_results_as_dict", "macro_sql": "{% macro default__get_query_results_as_dict(query) %}\n\n{# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #}\n\n {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%}\n\n {{ query }}\n\n {%- endcall -%}\n\n {% set sql_results={} %}\n\n {%- if execute -%}\n {% set sql_results_table = load_result('get_query_results').table.columns %}\n {% for column_name, column in sql_results_table.items() %}\n {% do sql_results.update({column_name: column.values()}) %}\n {% endfor %}\n {%- endif -%}\n\n {{ return(sql_results) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.110968}, "macro.dbt_utils.get_table_types_sql": {"unique_id": "macro.dbt_utils.get_table_types_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_table_types_sql.sql", "original_file_path": "macros/sql/get_table_types_sql.sql", "name": "get_table_types_sql", "macro_sql": "{%- macro get_table_types_sql() -%}\n {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__get_table_types_sql"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.111535}, "macro.dbt_utils.default__get_table_types_sql": {"unique_id": "macro.dbt_utils.default__get_table_types_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_table_types_sql.sql", "original_file_path": "macros/sql/get_table_types_sql.sql", "name": "default__get_table_types_sql", "macro_sql": "{% macro default__get_table_types_sql() %}\n case table_type\n when 'BASE TABLE' then 'table'\n when 'EXTERNAL TABLE' then 'external'\n when 'MATERIALIZED VIEW' then 'materializedview'\n else lower(table_type)\n end as \"table_type\"\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.111628}, "macro.dbt_utils.postgres__get_table_types_sql": {"unique_id": "macro.dbt_utils.postgres__get_table_types_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_table_types_sql.sql", "original_file_path": "macros/sql/get_table_types_sql.sql", "name": "postgres__get_table_types_sql", "macro_sql": "{% macro postgres__get_table_types_sql() %}\n case table_type\n when 'BASE TABLE' then 'table'\n when 'FOREIGN' then 'external'\n when 'MATERIALIZED VIEW' then 'materializedview'\n else lower(table_type)\n end as \"table_type\"\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1117141}, "macro.dbt_utils.bigquery__get_table_types_sql": {"unique_id": "macro.dbt_utils.bigquery__get_table_types_sql", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/get_table_types_sql.sql", "original_file_path": "macros/sql/get_table_types_sql.sql", "name": "bigquery__get_table_types_sql", "macro_sql": "{% macro bigquery__get_table_types_sql() %}\n case table_type\n when 'BASE TABLE' then 'table'\n when 'EXTERNAL TABLE' then 'external'\n when 'MATERIALIZED VIEW' then 'materializedview'\n else lower(table_type)\n end as `table_type`\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1118011}, "macro.dbt_utils.degrees_to_radians": {"unique_id": "macro.dbt_utils.degrees_to_radians", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/haversine_distance.sql", "original_file_path": "macros/sql/haversine_distance.sql", "name": "degrees_to_radians", "macro_sql": "{% macro degrees_to_radians(degrees) -%}\n acos(-1) * {{degrees}} / 180\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.112992}, "macro.dbt_utils.haversine_distance": {"unique_id": "macro.dbt_utils.haversine_distance", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/haversine_distance.sql", "original_file_path": "macros/sql/haversine_distance.sql", "name": "haversine_distance", "macro_sql": "{% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%}\n {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.bigquery__haversine_distance"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.113298}, "macro.dbt_utils.default__haversine_distance": {"unique_id": "macro.dbt_utils.default__haversine_distance", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/haversine_distance.sql", "original_file_path": "macros/sql/haversine_distance.sql", "name": "default__haversine_distance", "macro_sql": "{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%}\n{%- if unit == 'mi' %}\n {% set conversion_rate = 1 %}\n{% elif unit == 'km' %}\n {% set conversion_rate = 1.60934 %}\n{% else %}\n {{ exceptions.raise_compiler_error(\"unit input must be one of 'mi' or 'km'. Got \" ~ unit) }}\n{% endif %}\n\n 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) +\n cos(radians({{lat1}})) * cos(radians({{lat2}})) *\n power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1139169}, "macro.dbt_utils.bigquery__haversine_distance": {"unique_id": "macro.dbt_utils.bigquery__haversine_distance", "package_name": "dbt_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_utils", "path": "macros/sql/haversine_distance.sql", "original_file_path": "macros/sql/haversine_distance.sql", "name": "bigquery__haversine_distance", "macro_sql": "{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%}\n{% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %}\n{% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %}\n{% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %}\n{% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %}\n{%- if unit == 'mi' %}\n {% set conversion_rate = 1 %}\n{% elif unit == 'km' %}\n {% set conversion_rate = 1.60934 %}\n{% else %}\n {{ exceptions.raise_compiler_error(\"unit input must be one of 'mi' or 'km'. Got \" ~ unit) }}\n{% endif %}\n 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) +\n cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) *\n power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.degrees_to_radians"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.114877}, "macro.spark_utils.spark__type_numeric": {"unique_id": "macro.spark_utils.spark__type_numeric", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/dbt_utils/cross_db_utils/datatypes.sql", "original_file_path": "macros/dbt_utils/cross_db_utils/datatypes.sql", "name": "spark__type_numeric", "macro_sql": "{% macro spark__type_numeric() %}\n decimal(28, 6)\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.115094}, "macro.spark_utils.spark__dateadd": {"unique_id": "macro.spark_utils.spark__dateadd", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/dbt_utils/cross_db_utils/dateadd.sql", "original_file_path": "macros/dbt_utils/cross_db_utils/dateadd.sql", "name": "spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n to_unix_timestamp(to_timestamp({{from_date_or_timestamp}}))\n - to_unix_timestamp(date({{from_date_or_timestamp}}))\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n \n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(date({{from_date_or_timestamp}}), {{interval}} * {{multiplier}})\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n \n {%- set multiplier -%} \n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(date({{from_date_or_timestamp}}), {{interval}} * {{multiplier}})\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n \n {%- set multiplier -%} \n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp({{from_date_or_timestamp}})\n + {{interval}} * {{multiplier}}\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.117904}, "macro.spark_utils.spark__datediff": {"unique_id": "macro.spark_utils.spark__datediff", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/dbt_utils/cross_db_utils/datediff.sql", "original_file_path": "macros/dbt_utils/cross_db_utils/datediff.sql", "name": "spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart == 'day' -%}\n \n datediff({{second_date}}, {{first_date}})\n \n {%- elif datepart == 'week' -%}\n \n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n \n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n \n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n \n {%- elif datepart == 'quarter' -%}\n \n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n \n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n \n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n \n {%- set divisor -%} \n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n to_unix_timestamp({{second_date}}) \n - to_unix_timestamp({{first_date}})\n ) / {{divisor}})\n else floor((\n to_unix_timestamp({{second_date}}) \n - to_unix_timestamp({{first_date}})\n ) / {{divisor}})\n end\n \n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n \n {% if datepart == 'microsecond' %} \n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int) \n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.127518}, "macro.spark_utils.spark__current_timestamp": {"unique_id": "macro.spark_utils.spark__current_timestamp", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/dbt_utils/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/dbt_utils/cross_db_utils/current_timestamp.sql", "name": "spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() %}\n current_timestamp()\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1277509}, "macro.spark_utils.spark__current_timestamp_in_utc": {"unique_id": "macro.spark_utils.spark__current_timestamp_in_utc", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/dbt_utils/cross_db_utils/current_timestamp.sql", "original_file_path": "macros/dbt_utils/cross_db_utils/current_timestamp.sql", "name": "spark__current_timestamp_in_utc", "macro_sql": "{% macro spark__current_timestamp_in_utc() %}\n unix_timestamp()\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.127831}, "macro.spark_utils.spark__split_part": {"unique_id": "macro.spark_utils.spark__split_part", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/dbt_utils/cross_db_utils/split_part.sql", "original_file_path": "macros/dbt_utils/cross_db_utils/split_part.sql", "name": "spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n split(\n {{ string_text }},\n concat('\\\\', {{ delimiter_text }})\n )[({{ part_number - 1 }})]\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1281621}, "macro.spark_utils.spark__convert_timezone": {"unique_id": "macro.spark_utils.spark__convert_timezone", "package_name": "spark_utils", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/spark_utils", "path": "macros/snowplow/convert_timezone.sql", "original_file_path": "macros/snowplow/convert_timezone.sql", "name": "spark__convert_timezone", "macro_sql": "{% macro spark__convert_timezone(in_tz, out_tz, in_timestamp) %}\n from_utc_timestamp(to_utc_timestamp({{in_timestamp}}, {{in_tz}}), {{out_tz}})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.12846}, "macro.dbt_date.get_date_dimension": {"unique_id": "macro.dbt_date.get_date_dimension", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/get_date_dimension.sql", "original_file_path": "macros/get_date_dimension.sql", "name": "get_date_dimension", "macro_sql": "{% macro get_date_dimension(start_date, end_date) %}\n {{ adapter.dispatch('get_date_dimension', 'dbt_date') (start_date, end_date) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.default__get_date_dimension"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1373389}, "macro.dbt_date.default__get_date_dimension": {"unique_id": "macro.dbt_date.default__get_date_dimension", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/get_date_dimension.sql", "original_file_path": "macros/get_date_dimension.sql", "name": "default__get_date_dimension", "macro_sql": "{% macro default__get_date_dimension(start_date, end_date) %}\nwith base_dates as (\n {{ dbt_date.get_base_dates(start_date, end_date) }}\n),\ndates_with_prior_year_dates as (\n\n select\n cast(d.date_day as date) as date_day,\n cast({{ dbt_utils.dateadd('year', -1 , 'd.date_day') }} as date) as prior_year_date_day,\n cast({{ dbt_utils.dateadd('day', -364 , 'd.date_day') }} as date) as prior_year_over_year_date_day\n from\n \tbase_dates d\n\n)\nselect\n d.date_day,\n {{ dbt_date.yesterday('d.date_day') }} as prior_date_day,\n {{ dbt_date.tomorrow('d.date_day') }} as next_date_day,\n d.prior_year_date_day as prior_year_date_day,\n d.prior_year_over_year_date_day,\n {{ dbt_date.day_of_week('d.date_day', isoweek=false) }} as day_of_week,\n {{ dbt_date.day_of_week('d.date_day', isoweek=true) }} as day_of_week_iso,\n {{ dbt_date.day_name('d.date_day', short=false) }} as day_of_week_name,\n {{ dbt_date.day_name('d.date_day', short=true) }} as day_of_week_name_short,\n {{ dbt_date.day_of_month('d.date_day') }} as day_of_month,\n {{ dbt_date.day_of_year('d.date_day') }} as day_of_year,\n\n {{ dbt_date.week_start('d.date_day') }} as week_start_date,\n {{ dbt_date.week_end('d.date_day') }} as week_end_date,\n {{ dbt_date.week_start('d.prior_year_over_year_date_day') }} as prior_year_week_start_date,\n {{ dbt_date.week_end('d.prior_year_over_year_date_day') }} as prior_year_week_end_date,\n {{ dbt_date.week_of_year('d.date_day') }} as week_of_year,\n\n {{ dbt_date.iso_week_start('d.date_day') }} as iso_week_start_date,\n {{ dbt_date.iso_week_end('d.date_day') }} as iso_week_end_date,\n {{ dbt_date.iso_week_start('d.prior_year_over_year_date_day') }} as prior_year_iso_week_start_date,\n {{ dbt_date.iso_week_end('d.prior_year_over_year_date_day') }} as prior_year_iso_week_end_date,\n {{ dbt_date.iso_week_of_year('d.date_day') }} as iso_week_of_year,\n\n {{ dbt_date.week_of_year('d.prior_year_over_year_date_day') }} as prior_year_week_of_year,\n {{ dbt_date.iso_week_of_year('d.prior_year_over_year_date_day') }} as prior_year_iso_week_of_year,\n\n cast({{ dbt_date.date_part('month', 'd.date_day') }} as {{ dbt_utils.type_int() }}) as month_of_year,\n {{ dbt_date.month_name('d.date_day', short=false) }} as month_name,\n {{ dbt_date.month_name('d.date_day', short=true) }} as month_name_short,\n\n cast({{ dbt_utils.date_trunc('month', 'd.date_day') }} as date) as month_start_date,\n cast({{ dbt_utils.last_day('d.date_day', 'month') }} as date) as month_end_date,\n\n cast({{ dbt_utils.date_trunc('month', 'd.prior_year_date_day') }} as date) as prior_year_month_start_date,\n cast({{ dbt_utils.last_day('d.prior_year_date_day', 'month') }} as date) as prior_year_month_end_date,\n\n cast({{ dbt_date.date_part('quarter', 'd.date_day') }} as {{ dbt_utils.type_int() }}) as quarter_of_year,\n cast({{ dbt_utils.date_trunc('quarter', 'd.date_day') }} as date) as quarter_start_date,\n cast({{ dbt_utils.last_day('d.date_day', 'quarter') }} as date) as quarter_end_date,\n\n cast({{ dbt_date.date_part('year', 'd.date_day') }} as {{ dbt_utils.type_int() }}) as year_number,\n cast({{ dbt_utils.date_trunc('year', 'd.date_day') }} as date) as year_start_date,\n cast({{ dbt_utils.last_day('d.date_day', 'year') }} as date) as year_end_date\nfrom\n dates_with_prior_year_dates d\norder by 1\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.get_base_dates", "macro.dbt_utils.dateadd", "macro.dbt_date.yesterday", "macro.dbt_date.tomorrow", "macro.dbt_date.day_of_week", "macro.dbt_date.day_name", "macro.dbt_date.day_of_month", "macro.dbt_date.day_of_year", "macro.dbt_date.week_start", "macro.dbt_date.week_end", "macro.dbt_date.week_of_year", "macro.dbt_date.iso_week_start", "macro.dbt_date.iso_week_end", "macro.dbt_date.iso_week_of_year", "macro.dbt_date.date_part", "macro.dbt_utils.type_int", "macro.dbt_date.month_name", "macro.dbt_utils.date_trunc", "macro.dbt_utils.last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.139961}, "macro.dbt_date.postgres__get_date_dimension": {"unique_id": "macro.dbt_date.postgres__get_date_dimension", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/get_date_dimension.sql", "original_file_path": "macros/get_date_dimension.sql", "name": "postgres__get_date_dimension", "macro_sql": "{% macro postgres__get_date_dimension(start_date, end_date) %}\nwith base_dates as (\n {{ dbt_date.get_base_dates(start_date, end_date) }}\n),\ndates_with_prior_year_dates as (\n\n select\n cast(d.date_day as date) as date_day,\n cast({{ dbt_utils.dateadd('year', -1 , 'd.date_day') }} as date) as prior_year_date_day,\n cast({{ dbt_utils.dateadd('day', -364 , 'd.date_day') }} as date) as prior_year_over_year_date_day\n from\n \tbase_dates d\n\n)\nselect\n d.date_day,\n {{ dbt_date.yesterday('d.date_day') }} as prior_date_day,\n {{ dbt_date.tomorrow('d.date_day') }} as next_date_day,\n d.prior_year_date_day as prior_year_date_day,\n d.prior_year_over_year_date_day,\n {{ dbt_date.day_of_week('d.date_day', isoweek=true) }} as day_of_week,\n\n {{ dbt_date.day_name('d.date_day', short=false) }} as day_of_week_name,\n {{ dbt_date.day_name('d.date_day', short=true) }} as day_of_week_name_short,\n {{ dbt_date.day_of_month('d.date_day') }} as day_of_month,\n {{ dbt_date.day_of_year('d.date_day') }} as day_of_year,\n\n {{ dbt_date.week_start('d.date_day') }} as week_start_date,\n {{ dbt_date.week_end('d.date_day') }} as week_end_date,\n {{ dbt_date.week_start('d.prior_year_over_year_date_day') }} as prior_year_week_start_date,\n {{ dbt_date.week_end('d.prior_year_over_year_date_day') }} as prior_year_week_end_date,\n {{ dbt_date.week_of_year('d.date_day') }} as week_of_year,\n\n {{ dbt_date.iso_week_start('d.date_day') }} as iso_week_start_date,\n {{ dbt_date.iso_week_end('d.date_day') }} as iso_week_end_date,\n {{ dbt_date.iso_week_start('d.prior_year_over_year_date_day') }} as prior_year_iso_week_start_date,\n {{ dbt_date.iso_week_end('d.prior_year_over_year_date_day') }} as prior_year_iso_week_end_date,\n {{ dbt_date.iso_week_of_year('d.date_day') }} as iso_week_of_year,\n\n {{ dbt_date.week_of_year('d.prior_year_over_year_date_day') }} as prior_year_week_of_year,\n {{ dbt_date.iso_week_of_year('d.prior_year_over_year_date_day') }} as prior_year_iso_week_of_year,\n\n cast({{ dbt_date.date_part('month', 'd.date_day') }} as {{ dbt_utils.type_int() }}) as month_of_year,\n {{ dbt_date.month_name('d.date_day', short=false) }} as month_name,\n {{ dbt_date.month_name('d.date_day', short=true) }} as month_name_short,\n\n cast({{ dbt_utils.date_trunc('month', 'd.date_day') }} as date) as month_start_date,\n cast({{ dbt_utils.last_day('d.date_day', 'month') }} as date) as month_end_date,\n\n cast({{ dbt_utils.date_trunc('month', 'd.prior_year_date_day') }} as date) as prior_year_month_start_date,\n cast({{ dbt_utils.last_day('d.prior_year_date_day', 'month') }} as date) as prior_year_month_end_date,\n\n cast({{ dbt_date.date_part('quarter', 'd.date_day') }} as {{ dbt_utils.type_int() }}) as quarter_of_year,\n cast({{ dbt_utils.date_trunc('quarter', 'd.date_day') }} as date) as quarter_start_date,\n {# dbt_utils.last_day does not support quarter because postgresql does not support quarter interval. #}\n cast({{dbt_utils.dateadd('day', '-1', dbt_utils.dateadd('month', '3', dbt_utils.date_trunc('quarter', 'd.date_day')))}} as date) as quarter_end_date,\n\n cast({{ dbt_date.date_part('year', 'd.date_day') }} as {{ dbt_utils.type_int() }}) as year_number,\n cast({{ dbt_utils.date_trunc('year', 'd.date_day') }} as date) as year_start_date,\n cast({{ dbt_utils.last_day('d.date_day', 'year') }} as date) as year_end_date\nfrom\n dates_with_prior_year_dates d\norder by 1\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.get_base_dates", "macro.dbt_utils.dateadd", "macro.dbt_date.yesterday", "macro.dbt_date.tomorrow", "macro.dbt_date.day_of_week", "macro.dbt_date.day_name", "macro.dbt_date.day_of_month", "macro.dbt_date.day_of_year", "macro.dbt_date.week_start", "macro.dbt_date.week_end", "macro.dbt_date.week_of_year", "macro.dbt_date.iso_week_start", "macro.dbt_date.iso_week_end", "macro.dbt_date.iso_week_of_year", "macro.dbt_date.date_part", "macro.dbt_utils.type_int", "macro.dbt_date.month_name", "macro.dbt_utils.date_trunc", "macro.dbt_utils.last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1425831}, "macro.dbt_date.get_base_dates": {"unique_id": "macro.dbt_date.get_base_dates", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/get_base_dates.sql", "original_file_path": "macros/get_base_dates.sql", "name": "get_base_dates", "macro_sql": "{% macro get_base_dates(start_date=None, end_date=None, n_dateparts=None, datepart=\"day\") %}\n {{ adapter.dispatch('get_base_dates', 'dbt_date') (start_date, end_date, n_dateparts, datepart) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__get_base_dates"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.143724}, "macro.dbt_date.default__get_base_dates": {"unique_id": "macro.dbt_date.default__get_base_dates", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/get_base_dates.sql", "original_file_path": "macros/get_base_dates.sql", "name": "default__get_base_dates", "macro_sql": "{% macro default__get_base_dates(start_date, end_date, n_dateparts, datepart) %}\n\n{%- if start_date and end_date -%}\n{%- set start_date=\"cast('\" ~ start_date ~ \"' as \" ~ dbt_utils.type_timestamp() ~ \")\" -%}\n{%- set end_date=\"cast('\" ~ end_date ~ \"' as \" ~ dbt_utils.type_timestamp() ~ \")\" -%}\n\n{%- elif n_dateparts and datepart -%}\n\n{%- set start_date = dbt_utils.dateadd(datepart, -1 * n_dateparts, dbt_date.today()) -%}\n{%- set end_date = dbt_date.tomorrow() -%}\n{%- endif -%}\n\nwith date_spine as\n(\n\n {{ dbt_utils.date_spine(\n datepart=datepart,\n start_date=start_date,\n end_date=end_date,\n )\n }}\n\n)\nselect\n cast(d.date_{{ datepart }} as {{ dbt_utils.type_timestamp() }}) as date_{{ datepart }}\nfrom\n date_spine d\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_timestamp", "macro.dbt_utils.dateadd", "macro.dbt_date.today", "macro.dbt_date.tomorrow", "macro.dbt_utils.date_spine"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.144553}, "macro.dbt_date.bigquery__get_base_dates": {"unique_id": "macro.dbt_date.bigquery__get_base_dates", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/get_base_dates.sql", "original_file_path": "macros/get_base_dates.sql", "name": "bigquery__get_base_dates", "macro_sql": "{% macro bigquery__get_base_dates(start_date, end_date, n_dateparts, datepart) %}\n\n{%- if start_date and end_date -%}\n{%- set start_date=\"cast('\" ~ start_date ~ \"' as date )\" -%}\n{%- set end_date=\"cast('\" ~ end_date ~ \"' as date )\" -%}\n\n{%- elif n_dateparts and datepart -%}\n\n{%- set start_date = dbt_utils.dateadd(datepart, -1 * n_dateparts, dbt_date.today()) -%}\n{%- set end_date = dbt_date.tomorrow() -%}\n{%- endif -%}\n\nwith date_spine as\n(\n\n {{ dbt_utils.date_spine(\n datepart=datepart,\n start_date=start_date,\n end_date=end_date,\n )\n }}\n\n)\nselect\n cast(d.date_{{ datepart }} as {{ dbt_utils.type_timestamp() }}) as date_{{ datepart }}\nfrom\n date_spine d\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.dateadd", "macro.dbt_date.today", "macro.dbt_date.tomorrow", "macro.dbt_utils.date_spine", "macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1452968}, "macro.dbt_date.get_fiscal_year_dates": {"unique_id": "macro.dbt_date.get_fiscal_year_dates", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/fiscal_date/get_fiscal_year_dates.sql", "original_file_path": "macros/fiscal_date/get_fiscal_year_dates.sql", "name": "get_fiscal_year_dates", "macro_sql": "{% macro get_fiscal_year_dates(dates, year_end_month=12, week_start_day=1, shift_year=1) %}\n{{ adapter.dispatch('get_fiscal_year_dates', 'dbt_date') (dates, year_end_month, week_start_day, shift_year) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.default__get_fiscal_year_dates"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.147114}, "macro.dbt_date.default__get_fiscal_year_dates": {"unique_id": "macro.dbt_date.default__get_fiscal_year_dates", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/fiscal_date/get_fiscal_year_dates.sql", "original_file_path": "macros/fiscal_date/get_fiscal_year_dates.sql", "name": "default__get_fiscal_year_dates", "macro_sql": "{% macro default__get_fiscal_year_dates(dates, year_end_month, week_start_day, shift_year) %}\n-- this gets all the dates within a fiscal year\n-- determined by the given year-end-month\n-- ending on the saturday closest to that month's end date\nwith date_dimension as (\n select * from {{ dates }}\n),\nyear_month_end as (\n\n select\n d.year_number - {{ shift_year }} as fiscal_year_number,\n d.month_end_date\n from\n date_dimension d\n where\n d.month_of_year = {{ year_end_month }}\n group by 1,2\n\n),\nweeks as (\n\n select\n d.year_number,\n d.month_of_year,\n d.date_day as week_start_date,\n cast({{ dbt_utils.dateadd('day', 6, 'd.date_day') }} as date) as week_end_date\n from\n date_dimension d\n where\n d.day_of_week = {{ week_start_day }}\n\n),\n-- get all the weeks that start in the month the year ends\nyear_week_ends as (\n\n select\n d.year_number - {{ shift_year }} as fiscal_year_number,\n d.week_end_date\n from\n weeks d\n where\n d.month_of_year = {{ year_end_month }}\n group by\n 1,2\n\n),\n-- then calculate which Saturday is closest to month end\nweeks_at_month_end as (\n\n select\n d.fiscal_year_number,\n d.week_end_date,\n m.month_end_date,\n rank() over\n (partition by d.fiscal_year_number\n order by\n abs({{ dbt_utils.datediff('d.week_end_date', 'm.month_end_date', 'day') }})\n\n ) as closest_to_month_end\n from\n year_week_ends d\n join\n year_month_end m on d.fiscal_year_number = m.fiscal_year_number\n),\nfiscal_year_range as (\n\n select\n w.fiscal_year_number,\n cast(\n {{ dbt_utils.dateadd('day', 1,\n 'lag(w.week_end_date) over(order by w.week_end_date)') }}\n as date) as fiscal_year_start_date,\n w.week_end_date as fiscal_year_end_date\n from\n weeks_at_month_end w\n where\n w.closest_to_month_end = 1\n\n),\nfiscal_year_dates as (\n\n select\n d.date_day,\n m.fiscal_year_number,\n m.fiscal_year_start_date,\n m.fiscal_year_end_date,\n w.week_start_date,\n w.week_end_date,\n -- we reset the weeks of the year starting with the merch year start date\n dense_rank()\n over(\n partition by m.fiscal_year_number\n order by w.week_start_date\n ) as fiscal_week_of_year\n from\n date_dimension d\n join\n fiscal_year_range m on d.date_day between m.fiscal_year_start_date and m.fiscal_year_end_date\n join\n weeks w on d.date_day between w.week_start_date and w.week_end_date\n\n)\nselect * from fiscal_year_dates order by 1\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.dateadd", "macro.dbt_utils.datediff"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.147753}, "macro.dbt_date.get_fiscal_periods": {"unique_id": "macro.dbt_date.get_fiscal_periods", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/fiscal_date/get_fiscal_periods.sql", "original_file_path": "macros/fiscal_date/get_fiscal_periods.sql", "name": "get_fiscal_periods", "macro_sql": "{% macro get_fiscal_periods(dates, year_end_month, week_start_day, shift_year=1) %}\n{# \nThis macro requires you to pass in a ref to a date dimension, created via\ndbt_date.get_date_dimension()s\n#}\nwith fscl_year_dates_for_periods as (\n {{ dbt_date.get_fiscal_year_dates(dates, year_end_month, week_start_day, shift_year) }}\n),\nfscl_year_w13 as (\n\n select\n f.*,\n -- We count the weeks in a 13 week period\n -- and separate the 4-5-4 week sequences\n mod(cast(\n (f.fiscal_week_of_year-1) as {{ dbt_utils.type_int() }}\n ), 13) as w13_number,\n -- Chop weeks into 13 week merch quarters\n cast(\n least(\n floor((f.fiscal_week_of_year-1)/13.0)\n , 3)\n as {{ dbt_utils.type_int() }}) as quarter_number\n from\n fscl_year_dates_for_periods f\n\n),\nfscl_periods as (\n\n select\n f.date_day,\n f.fiscal_year_number,\n f.week_start_date,\n f.week_end_date,\n f.fiscal_week_of_year,\n case \n -- we move week 53 into the 3rd period of the quarter\n when f.fiscal_week_of_year = 53 then 3\n when f.w13_number between 0 and 3 then 1\n when f.w13_number between 4 and 8 then 2\n when f.w13_number between 9 and 12 then 3\n end as period_of_quarter,\n f.quarter_number\n from\n fscl_year_w13 f\n\n),\nfscl_periods_quarters as (\n\n select\n f.*,\n cast((\n (f.quarter_number * 3) + f.period_of_quarter\n ) as {{ dbt_utils.type_int() }}) as fiscal_period_number\n from\n fscl_periods f\n\n)\nselect\n date_day,\n fiscal_year_number,\n week_start_date,\n week_end_date,\n fiscal_week_of_year, \n dense_rank() over(partition by fiscal_period_number order by fiscal_week_of_year) as fiscal_week_of_period,\n fiscal_period_number,\n quarter_number+1 as fiscal_quarter_number,\n period_of_quarter as fiscal_period_of_quarter\nfrom \n fscl_periods_quarters \norder by 1,2\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.get_fiscal_year_dates", "macro.dbt_utils.type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.14878}, "macro.dbt_date.tomorrow": {"unique_id": "macro.dbt_date.tomorrow", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/tomorrow.sql", "original_file_path": "macros/calendar_date/tomorrow.sql", "name": "tomorrow", "macro_sql": "{%- macro tomorrow(date=None, tz=None) -%}\n{{ dbt_date.n_days_away(1, date, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_days_away"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.149096}, "macro.dbt_date.next_week": {"unique_id": "macro.dbt_date.next_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/next_week.sql", "original_file_path": "macros/calendar_date/next_week.sql", "name": "next_week", "macro_sql": "{%- macro next_week(tz=None) -%}\n{{ dbt_date.n_weeks_away(1, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_weeks_away"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1493719}, "macro.dbt_date.next_month_name": {"unique_id": "macro.dbt_date.next_month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/next_month_name.sql", "original_file_path": "macros/calendar_date/next_month_name.sql", "name": "next_month_name", "macro_sql": "{%- macro next_month_name(short=True, tz=None) -%}\n{{ dbt_date.month_name(dbt_date.next_month(1, tz), short=short) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.month_name", "macro.dbt_date.next_month"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.149729}, "macro.dbt_date.next_month": {"unique_id": "macro.dbt_date.next_month", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/next_month.sql", "original_file_path": "macros/calendar_date/next_month.sql", "name": "next_month", "macro_sql": "{%- macro next_month(tz=None) -%}\n{{ dbt_date.n_months_away(1, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_months_away"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.150018}, "macro.dbt_date.day_name": {"unique_id": "macro.dbt_date.day_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_name.sql", "original_file_path": "macros/calendar_date/day_name.sql", "name": "day_name", "macro_sql": "{%- macro day_name(date, short=True) -%}\n {{ adapter.dispatch('day_name', 'dbt_date') (date, short) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__day_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.150778}, "macro.dbt_date.default__day_name": {"unique_id": "macro.dbt_date.default__day_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_name.sql", "original_file_path": "macros/calendar_date/day_name.sql", "name": "default__day_name", "macro_sql": "\n\n{%- macro default__day_name(date, short) -%}\n{%- set f = 'Dy' if short else 'Day' -%}\n to_char({{ date }}, '{{ f }}')\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1509929}, "macro.dbt_date.snowflake__day_name": {"unique_id": "macro.dbt_date.snowflake__day_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_name.sql", "original_file_path": "macros/calendar_date/day_name.sql", "name": "snowflake__day_name", "macro_sql": "\n\n{%- macro snowflake__day_name(date, short) -%}\n {%- if short -%}\n dayname({{ date }})\n {%- else -%}\n -- long version not implemented on Snowflake so we're doing it manually :/\n case dayname({{ date }})\n when 'Mon' then 'Monday'\n when 'Tue' then 'Tuesday'\n when 'Wed' then 'Wednesday'\n when 'Thu' then 'Thursday'\n when 'Fri' then 'Friday'\n when 'Sat' then 'Saturday'\n when 'Sun' then 'Sunday'\n end\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.151215}, "macro.dbt_date.bigquery__day_name": {"unique_id": "macro.dbt_date.bigquery__day_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_name.sql", "original_file_path": "macros/calendar_date/day_name.sql", "name": "bigquery__day_name", "macro_sql": "\n\n{%- macro bigquery__day_name(date, short) -%}\n{%- set f = '%a' if short else '%A' -%}\n format_date('{{ f }}', cast({{ date }} as date))\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1514308}, "macro.dbt_date.postgres__day_name": {"unique_id": "macro.dbt_date.postgres__day_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_name.sql", "original_file_path": "macros/calendar_date/day_name.sql", "name": "postgres__day_name", "macro_sql": "\n\n{%- macro postgres__day_name(date, short) -%}\n{# FM = Fill mode, which suppresses padding blanks #}\n{%- set f = 'FMDy' if short else 'FMDay' -%}\n to_char({{ date }}, '{{ f }}')\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.151654}, "macro.dbt_date.to_unixtimestamp": {"unique_id": "macro.dbt_date.to_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/to_unixtimestamp.sql", "original_file_path": "macros/calendar_date/to_unixtimestamp.sql", "name": "to_unixtimestamp", "macro_sql": "{%- macro to_unixtimestamp(timestamp) -%}\n {{ adapter.dispatch('to_unixtimestamp', 'dbt_date') (timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__to_unixtimestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.152065}, "macro.dbt_date.default__to_unixtimestamp": {"unique_id": "macro.dbt_date.default__to_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/to_unixtimestamp.sql", "original_file_path": "macros/calendar_date/to_unixtimestamp.sql", "name": "default__to_unixtimestamp", "macro_sql": "\n\n{%- macro default__to_unixtimestamp(timestamp) -%}\n {{ dbt_date.date_part('epoch', timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1522272}, "macro.dbt_date.snowflake__to_unixtimestamp": {"unique_id": "macro.dbt_date.snowflake__to_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/to_unixtimestamp.sql", "original_file_path": "macros/calendar_date/to_unixtimestamp.sql", "name": "snowflake__to_unixtimestamp", "macro_sql": "\n\n{%- macro snowflake__to_unixtimestamp(timestamp) -%}\n {{ dbt_date.date_part('epoch_seconds', timestamp) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1523762}, "macro.dbt_date.bigquery__to_unixtimestamp": {"unique_id": "macro.dbt_date.bigquery__to_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/to_unixtimestamp.sql", "original_file_path": "macros/calendar_date/to_unixtimestamp.sql", "name": "bigquery__to_unixtimestamp", "macro_sql": "\n\n{%- macro bigquery__to_unixtimestamp(timestamp) -%}\n unix_seconds({{ timestamp }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.152483}, "macro.dbt_date.n_days_away": {"unique_id": "macro.dbt_date.n_days_away", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/n_days_away.sql", "original_file_path": "macros/calendar_date/n_days_away.sql", "name": "n_days_away", "macro_sql": "{%- macro n_days_away(n, date=None, tz=None) -%}\n{{ dbt_date.n_days_ago(-1 * n, date, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_days_ago"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.152891}, "macro.dbt_date.week_start": {"unique_id": "macro.dbt_date.week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_start.sql", "original_file_path": "macros/calendar_date/week_start.sql", "name": "week_start", "macro_sql": "{%- macro week_start(date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{{ adapter.dispatch('week_start', 'dbt_date') (dt) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_date.default__week_start"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.153475}, "macro.dbt_date.default__week_start": {"unique_id": "macro.dbt_date.default__week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_start.sql", "original_file_path": "macros/calendar_date/week_start.sql", "name": "default__week_start", "macro_sql": "{%- macro default__week_start(date) -%}\ncast({{ dbt_utils.date_trunc('week', date) }} as date)\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.153633}, "macro.dbt_date.snowflake__week_start": {"unique_id": "macro.dbt_date.snowflake__week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_start.sql", "original_file_path": "macros/calendar_date/week_start.sql", "name": "snowflake__week_start", "macro_sql": "\n\n{%- macro snowflake__week_start(date) -%}\n {#\n Get the day of week offset: e.g. if the date is a Sunday,\n dbt_date.day_of_week returns 1, so we subtract 1 to get a 0 offset\n #}\n {% set off_set = dbt_date.day_of_week(date, isoweek=False) ~ \" - 1\" %}\n cast({{ dbt_utils.dateadd(\"day\", \"-1 * (\" ~ off_set ~ \")\", date) }} as date)\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.day_of_week", "macro.dbt_utils.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.153952}, "macro.dbt_date.postgres__week_start": {"unique_id": "macro.dbt_date.postgres__week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_start.sql", "original_file_path": "macros/calendar_date/week_start.sql", "name": "postgres__week_start", "macro_sql": "\n\n{%- macro postgres__week_start(date) -%}\n-- Sunday as week start date\ncast({{ dbt_utils.dateadd('day', -1, dbt_utils.date_trunc('week', dbt_utils.dateadd('day', 1, date))) }} as date)\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.dateadd", "macro.dbt_utils.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.154239}, "macro.dbt_date.iso_week_start": {"unique_id": "macro.dbt_date.iso_week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_start.sql", "original_file_path": "macros/calendar_date/iso_week_start.sql", "name": "iso_week_start", "macro_sql": "{%- macro iso_week_start(date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{{ adapter.dispatch('iso_week_start', 'dbt_date') (dt) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_date.default__iso_week_start"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.154855}, "macro.dbt_date._iso_week_start": {"unique_id": "macro.dbt_date._iso_week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_start.sql", "original_file_path": "macros/calendar_date/iso_week_start.sql", "name": "_iso_week_start", "macro_sql": "{%- macro _iso_week_start(date, week_type) -%}\ncast({{ dbt_utils.date_trunc(week_type, date) }} as date)\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.15502}, "macro.dbt_date.default__iso_week_start": {"unique_id": "macro.dbt_date.default__iso_week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_start.sql", "original_file_path": "macros/calendar_date/iso_week_start.sql", "name": "default__iso_week_start", "macro_sql": "\n\n{%- macro default__iso_week_start(date) -%}\n{{ dbt_date._iso_week_start(date, 'isoweek') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_start"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.155168}, "macro.dbt_date.snowflake__iso_week_start": {"unique_id": "macro.dbt_date.snowflake__iso_week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_start.sql", "original_file_path": "macros/calendar_date/iso_week_start.sql", "name": "snowflake__iso_week_start", "macro_sql": "\n\n{%- macro snowflake__iso_week_start(date) -%}\n{{ dbt_date._iso_week_start(date, 'week') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_start"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1553159}, "macro.dbt_date.postgres__iso_week_start": {"unique_id": "macro.dbt_date.postgres__iso_week_start", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_start.sql", "original_file_path": "macros/calendar_date/iso_week_start.sql", "name": "postgres__iso_week_start", "macro_sql": "\n\n{%- macro postgres__iso_week_start(date) -%}\n{{ dbt_date._iso_week_start(date, 'week') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_start"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.155463}, "macro.dbt_date.n_days_ago": {"unique_id": "macro.dbt_date.n_days_ago", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/n_days_ago.sql", "original_file_path": "macros/calendar_date/n_days_ago.sql", "name": "n_days_ago", "macro_sql": "{%- macro n_days_ago(n, date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{%- set n = n|int -%}\ncast({{ dbt_utils.dateadd('day', -1 * n, dt) }} as date)\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_utils.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.155991}, "macro.dbt_date.last_week": {"unique_id": "macro.dbt_date.last_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/last_week.sql", "original_file_path": "macros/calendar_date/last_week.sql", "name": "last_week", "macro_sql": "{%- macro last_week(tz=None) -%}\n{{ dbt_date.n_weeks_ago(1, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_weeks_ago"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.156266}, "macro.dbt_date.now": {"unique_id": "macro.dbt_date.now", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/now.sql", "original_file_path": "macros/calendar_date/now.sql", "name": "now", "macro_sql": "{%- macro now(tz=None) -%}\n{{ dbt_date.convert_timezone(dbt_utils.current_timestamp(), tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.convert_timezone", "macro.dbt_utils.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.156559}, "macro.dbt_date.periods_since": {"unique_id": "macro.dbt_date.periods_since", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/periods_since.sql", "original_file_path": "macros/calendar_date/periods_since.sql", "name": "periods_since", "macro_sql": "{%- macro periods_since(date_col, period_name='day', tz=None) -%}\n{{ dbt_utils.datediff(date_col, dbt_date.now(tz), period_name) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.datediff", "macro.dbt_date.now"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.15692}, "macro.dbt_date.today": {"unique_id": "macro.dbt_date.today", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/today.sql", "original_file_path": "macros/calendar_date/today.sql", "name": "today", "macro_sql": "{%- macro today(tz=None) -%}\ncast({{ dbt_date.now(tz) }} as date)\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.now"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.157185}, "macro.dbt_date.last_month": {"unique_id": "macro.dbt_date.last_month", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/last_month.sql", "original_file_path": "macros/calendar_date/last_month.sql", "name": "last_month", "macro_sql": "{%- macro last_month(tz=None) -%}\n{{ dbt_date.n_months_ago(1, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_months_ago"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.157454}, "macro.dbt_date.day_of_year": {"unique_id": "macro.dbt_date.day_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_year.sql", "original_file_path": "macros/calendar_date/day_of_year.sql", "name": "day_of_year", "macro_sql": "{%- macro day_of_year(date) -%}\n{{ adapter.dispatch('day_of_year', 'dbt_date') (date) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.default__day_of_year"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.157869}, "macro.dbt_date.default__day_of_year": {"unique_id": "macro.dbt_date.default__day_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_year.sql", "original_file_path": "macros/calendar_date/day_of_year.sql", "name": "default__day_of_year", "macro_sql": "\n\n{%- macro default__day_of_year(date) -%}\n {{ dbt_date.date_part('dayofyear', date) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.158016}, "macro.dbt_date.postgres__day_of_year": {"unique_id": "macro.dbt_date.postgres__day_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_year.sql", "original_file_path": "macros/calendar_date/day_of_year.sql", "name": "postgres__day_of_year", "macro_sql": "\n\n{%- macro postgres__day_of_year(date) -%}\n {{ dbt_date.date_part('doy', date) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.15816}, "macro.dbt_date.redshift__day_of_year": {"unique_id": "macro.dbt_date.redshift__day_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_year.sql", "original_file_path": "macros/calendar_date/day_of_year.sql", "name": "redshift__day_of_year", "macro_sql": "\n\n{%- macro redshift__day_of_year(date) -%}\n cast({{ dbt_date.date_part('dayofyear', date) }} as {{ dbt_utils.type_bigint() }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_utils.type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.158358}, "macro.dbt_date.from_unixtimestamp": {"unique_id": "macro.dbt_date.from_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/from_unixtimestamp.sql", "original_file_path": "macros/calendar_date/from_unixtimestamp.sql", "name": "from_unixtimestamp", "macro_sql": "{%- macro from_unixtimestamp(epochs, format=\"seconds\") -%}\n {{ adapter.dispatch('from_unixtimestamp', 'dbt_date') (epochs, format) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__from_unixtimestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.159754}, "macro.dbt_date.default__from_unixtimestamp": {"unique_id": "macro.dbt_date.default__from_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/from_unixtimestamp.sql", "original_file_path": "macros/calendar_date/from_unixtimestamp.sql", "name": "default__from_unixtimestamp", "macro_sql": "\n\n{%- macro default__from_unixtimestamp(epochs, format=\"seconds\") -%}\n {%- if format != \"seconds\" -%}\n {{ exceptions.raise_compiler_error(\n \"value \" ~ format ~ \" for `format` for from_unixtimestamp is not supported.\"\n )\n }}\n {% endif -%}\n to_timestamp({{ epochs }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.160032}, "macro.dbt_date.postgres__from_unixtimestamp": {"unique_id": "macro.dbt_date.postgres__from_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/from_unixtimestamp.sql", "original_file_path": "macros/calendar_date/from_unixtimestamp.sql", "name": "postgres__from_unixtimestamp", "macro_sql": "\n\n{%- macro postgres__from_unixtimestamp(epochs, format=\"seconds\") -%}\n {%- if format != \"seconds\" -%}\n {{ exceptions.raise_compiler_error(\n \"value \" ~ format ~ \" for `format` for from_unixtimestamp is not supported.\"\n )\n }}\n {% endif -%}\n cast(to_timestamp({{ epochs }}) at time zone 'UTC' as timestamp)\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.160316}, "macro.dbt_date.snowflake__from_unixtimestamp": {"unique_id": "macro.dbt_date.snowflake__from_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/from_unixtimestamp.sql", "original_file_path": "macros/calendar_date/from_unixtimestamp.sql", "name": "snowflake__from_unixtimestamp", "macro_sql": "\n\n{%- macro snowflake__from_unixtimestamp(epochs, format) -%}\n {%- if format == \"seconds\" -%}\n {%- set scale = 0 -%}\n {%- elif format == \"milliseconds\" -%}\n {%- set scale = 3 -%}\n {%- elif format == \"microseconds\" -%}\n {%- set scale = 6 -%}\n {%- else -%}\n {{ exceptions.raise_compiler_error(\n \"value \" ~ format ~ \" for `format` for from_unixtimestamp is not supported.\"\n )\n }}\n {% endif -%}\n to_timestamp_ntz({{ epochs }}, {{ scale }})\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.160823}, "macro.dbt_date.bigquery__from_unixtimestamp": {"unique_id": "macro.dbt_date.bigquery__from_unixtimestamp", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/from_unixtimestamp.sql", "original_file_path": "macros/calendar_date/from_unixtimestamp.sql", "name": "bigquery__from_unixtimestamp", "macro_sql": "\n\n{%- macro bigquery__from_unixtimestamp(epochs, format) -%}\n {%- if format == \"seconds\" -%}\n timestamp_seconds({{ epochs }})\n {%- elif format == \"milliseconds\" -%}\n timestamp_millis({{ epochs }})\n {%- elif format == \"microseconds\" -%}\n timestamp_micros({{ epochs }})\n {%- else -%}\n {{ exceptions.raise_compiler_error(\n \"value \" ~ format ~ \" for `format` for from_unixtimestamp is not supported.\"\n )\n }}\n {% endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.161247}, "macro.dbt_date.n_months_ago": {"unique_id": "macro.dbt_date.n_months_ago", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/n_months_ago.sql", "original_file_path": "macros/calendar_date/n_months_ago.sql", "name": "n_months_ago", "macro_sql": "{%- macro n_months_ago(n, tz=None) -%}\n{%- set n = n|int -%}\n{{ dbt_utils.date_trunc('month', \n dbt_utils.dateadd('month', -1 * n, \n dbt_date.today(tz)\n )\n ) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc", "macro.dbt_utils.dateadd", "macro.dbt_date.today"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.161741}, "macro.dbt_date.date_part": {"unique_id": "macro.dbt_date.date_part", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/date_part.sql", "original_file_path": "macros/calendar_date/date_part.sql", "name": "date_part", "macro_sql": "{% macro date_part(datepart, date) -%}\n {{ adapter.dispatch('date_part', 'dbt_date') (datepart, date) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.162144}, "macro.dbt_date.default__date_part": {"unique_id": "macro.dbt_date.default__date_part", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/date_part.sql", "original_file_path": "macros/calendar_date/date_part.sql", "name": "default__date_part", "macro_sql": "{% macro default__date_part(datepart, date) -%}\n date_part('{{ datepart }}', {{ date }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1622882}, "macro.dbt_date.bigquery__date_part": {"unique_id": "macro.dbt_date.bigquery__date_part", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/date_part.sql", "original_file_path": "macros/calendar_date/date_part.sql", "name": "bigquery__date_part", "macro_sql": "{% macro bigquery__date_part(datepart, date) -%}\n extract({{ datepart }} from {{ date }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.162427}, "macro.dbt_date.n_weeks_away": {"unique_id": "macro.dbt_date.n_weeks_away", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/n_weeks_away.sql", "original_file_path": "macros/calendar_date/n_weeks_away.sql", "name": "n_weeks_away", "macro_sql": "{%- macro n_weeks_away(n, tz=None) -%}\n{%- set n = n|int -%}\n{{ dbt_utils.date_trunc('week', \n dbt_utils.dateadd('week', n, \n dbt_date.today(tz)\n )\n ) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc", "macro.dbt_utils.dateadd", "macro.dbt_date.today"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.162883}, "macro.dbt_date.day_of_month": {"unique_id": "macro.dbt_date.day_of_month", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_month.sql", "original_file_path": "macros/calendar_date/day_of_month.sql", "name": "day_of_month", "macro_sql": "{%- macro day_of_month(date) -%}\n{{ dbt_date.date_part('day', date) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1631858}, "macro.dbt_date.redshift__day_of_month": {"unique_id": "macro.dbt_date.redshift__day_of_month", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_month.sql", "original_file_path": "macros/calendar_date/day_of_month.sql", "name": "redshift__day_of_month", "macro_sql": "\n\n{%- macro redshift__day_of_month(date) -%}\ncast({{ dbt_date.date_part('day', date) }} as {{ dbt_utils.type_bigint() }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_utils.type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.163384}, "macro.dbt_date.yesterday": {"unique_id": "macro.dbt_date.yesterday", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/yesterday.sql", "original_file_path": "macros/calendar_date/yesterday.sql", "name": "yesterday", "macro_sql": "{%- macro yesterday(date=None, tz=None) -%}\n{{ dbt_date.n_days_ago(1, date, tz) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.n_days_ago"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.163698}, "macro.dbt_date.day_of_week": {"unique_id": "macro.dbt_date.day_of_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_week.sql", "original_file_path": "macros/calendar_date/day_of_week.sql", "name": "day_of_week", "macro_sql": "{%- macro day_of_week(date, isoweek=true) -%}\n{{ adapter.dispatch('day_of_week', 'dbt_date') (date, isoweek) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__day_of_week"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1656058}, "macro.dbt_date.default__day_of_week": {"unique_id": "macro.dbt_date.default__day_of_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_week.sql", "original_file_path": "macros/calendar_date/day_of_week.sql", "name": "default__day_of_week", "macro_sql": "\n\n{%- macro default__day_of_week(date, isoweek) -%}\n\n {%- set dow = dbt_date.date_part('dayofweek', date) -%}\n\n {%- if isoweek -%}\n case\n -- Shift start of week from Sunday (0) to Monday (1)\n when {{ dow }} = 0 then 7\n else {{ dow }}\n end\n {%- else -%}\n {{ dow }} + 1\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.165919}, "macro.dbt_date.snowflake__day_of_week": {"unique_id": "macro.dbt_date.snowflake__day_of_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_week.sql", "original_file_path": "macros/calendar_date/day_of_week.sql", "name": "snowflake__day_of_week", "macro_sql": "\n\n{%- macro snowflake__day_of_week(date, isoweek) -%}\n\n {%- if isoweek -%}\n {%- set dow_part = 'dayofweekiso' -%}\n {{ dbt_date.date_part(dow_part, date) }}\n {%- else -%}\n {%- set dow_part = 'dayofweek' -%}\n case\n when {{ dbt_date.date_part(dow_part, date) }} = 7 then 1\n else {{ dbt_date.date_part(dow_part, date) }} + 1\n end\n {%- endif -%}\n\n\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.166362}, "macro.dbt_date.bigquery__day_of_week": {"unique_id": "macro.dbt_date.bigquery__day_of_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_week.sql", "original_file_path": "macros/calendar_date/day_of_week.sql", "name": "bigquery__day_of_week", "macro_sql": "\n\n{%- macro bigquery__day_of_week(date, isoweek) -%}\n\n {%- set dow = dbt_date.date_part('dayofweek', date) -%}\n\n {%- if isoweek -%}\n case\n -- Shift start of week from Sunday (1) to Monday (2)\n when {{ dow }} = 1 then 7\n else {{ dow }} - 1\n end\n {%- else -%}\n {{ dow }}\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.166667}, "macro.dbt_date.postgres__day_of_week": {"unique_id": "macro.dbt_date.postgres__day_of_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_week.sql", "original_file_path": "macros/calendar_date/day_of_week.sql", "name": "postgres__day_of_week", "macro_sql": "\n\n\n{%- macro postgres__day_of_week(date, isoweek) -%}\n\n {%- if isoweek -%}\n {%- set dow_part = 'isodow' -%}\n -- Monday(1) to Sunday (7)\n cast({{ dbt_date.date_part(dow_part, date) }} as {{ dbt_utils.type_int() }})\n {%- else -%}\n {%- set dow_part = 'dow' -%}\n -- Sunday(1) to Saturday (7)\n cast({{ dbt_date.date_part(dow_part, date) }} + 1 as {{ dbt_utils.type_int() }})\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_utils.type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.16713}, "macro.dbt_date.redshift__day_of_week": {"unique_id": "macro.dbt_date.redshift__day_of_week", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/day_of_week.sql", "original_file_path": "macros/calendar_date/day_of_week.sql", "name": "redshift__day_of_week", "macro_sql": "\n\n\n{%- macro redshift__day_of_week(date, isoweek) -%}\n\n {%- set dow = dbt_date.date_part('dayofweek', date) -%}\n\n {%- if isoweek -%}\n case\n -- Shift start of week from Sunday (0) to Monday (1)\n when {{ dow }} = 0 then 7\n else cast({{ dow }} as {{ dbt_utils.type_bigint() }})\n end\n {%- else -%}\n cast({{ dow }} + 1 as {{ dbt_utils.type_bigint() }})\n {%- endif -%}\n\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_utils.type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.167595}, "macro.dbt_date.iso_week_end": {"unique_id": "macro.dbt_date.iso_week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_end.sql", "original_file_path": "macros/calendar_date/iso_week_end.sql", "name": "iso_week_end", "macro_sql": "{%- macro iso_week_end(date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{{ adapter.dispatch('iso_week_end', 'dbt_date') (dt) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_date.default__iso_week_end"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.168171}, "macro.dbt_date._iso_week_end": {"unique_id": "macro.dbt_date._iso_week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_end.sql", "original_file_path": "macros/calendar_date/iso_week_end.sql", "name": "_iso_week_end", "macro_sql": "{%- macro _iso_week_end(date, week_type) -%}\n{%- set dt = dbt_date.iso_week_start(date) -%}\n{{ dbt_date.n_days_away(6, dt) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.iso_week_start", "macro.dbt_date.n_days_away"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1683998}, "macro.dbt_date.default__iso_week_end": {"unique_id": "macro.dbt_date.default__iso_week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_end.sql", "original_file_path": "macros/calendar_date/iso_week_end.sql", "name": "default__iso_week_end", "macro_sql": "\n\n{%- macro default__iso_week_end(date) -%}\n{{ dbt_date._iso_week_end(date, 'isoweek') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_end"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1685472}, "macro.dbt_date.snowflake__iso_week_end": {"unique_id": "macro.dbt_date.snowflake__iso_week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_end.sql", "original_file_path": "macros/calendar_date/iso_week_end.sql", "name": "snowflake__iso_week_end", "macro_sql": "\n\n{%- macro snowflake__iso_week_end(date) -%}\n{{ dbt_date._iso_week_end(date, 'weekiso') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_end"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.168691}, "macro.dbt_date.n_weeks_ago": {"unique_id": "macro.dbt_date.n_weeks_ago", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/n_weeks_ago.sql", "original_file_path": "macros/calendar_date/n_weeks_ago.sql", "name": "n_weeks_ago", "macro_sql": "{%- macro n_weeks_ago(n, tz=None) -%}\n{%- set n = n|int -%}\n{{ dbt_utils.date_trunc('week', \n dbt_utils.dateadd('week', -1 * n, \n dbt_date.today(tz)\n )\n ) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc", "macro.dbt_utils.dateadd", "macro.dbt_date.today"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.169179}, "macro.dbt_date.month_name": {"unique_id": "macro.dbt_date.month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/month_name.sql", "original_file_path": "macros/calendar_date/month_name.sql", "name": "month_name", "macro_sql": "{%- macro month_name(date, short=True) -%}\n {{ adapter.dispatch('month_name', 'dbt_date') (date, short) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__month_name"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.169759}, "macro.dbt_date.default__month_name": {"unique_id": "macro.dbt_date.default__month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/month_name.sql", "original_file_path": "macros/calendar_date/month_name.sql", "name": "default__month_name", "macro_sql": "\n\n{%- macro default__month_name(date, short) -%}\n{%- set f = 'MON' if short else 'MONTH' -%}\n to_char({{ date }}, '{{ f }}')\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1699781}, "macro.dbt_date.bigquery__month_name": {"unique_id": "macro.dbt_date.bigquery__month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/month_name.sql", "original_file_path": "macros/calendar_date/month_name.sql", "name": "bigquery__month_name", "macro_sql": "\n\n{%- macro bigquery__month_name(date, short) -%}\n{%- set f = '%b' if short else '%B' -%}\n format_date('{{ f }}', cast({{ date }} as date))\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.170191}, "macro.dbt_date.snowflake__month_name": {"unique_id": "macro.dbt_date.snowflake__month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/month_name.sql", "original_file_path": "macros/calendar_date/month_name.sql", "name": "snowflake__month_name", "macro_sql": "\n\n{%- macro snowflake__month_name(date, short) -%}\n{%- set f = 'MON' if short else 'MMMM' -%}\n to_char({{ date }}, '{{ f }}')\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1704051}, "macro.dbt_date.postgres__month_name": {"unique_id": "macro.dbt_date.postgres__month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/month_name.sql", "original_file_path": "macros/calendar_date/month_name.sql", "name": "postgres__month_name", "macro_sql": "\n\n{%- macro postgres__month_name(date, short) -%}\n{# FM = Fill mode, which suppresses padding blanks #}\n{%- set f = 'FMMon' if short else 'FMMonth' -%}\n to_char({{ date }}, '{{ f }}')\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.170624}, "macro.dbt_date.last_month_name": {"unique_id": "macro.dbt_date.last_month_name", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/last_month_name.sql", "original_file_path": "macros/calendar_date/last_month_name.sql", "name": "last_month_name", "macro_sql": "{%- macro last_month_name(short=True, tz=None) -%}\n{{ dbt_date.month_name(dbt_date.last_month(1, tz), short=short) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.month_name", "macro.dbt_date.last_month"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1709812}, "macro.dbt_date.week_of_year": {"unique_id": "macro.dbt_date.week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_of_year.sql", "original_file_path": "macros/calendar_date/week_of_year.sql", "name": "week_of_year", "macro_sql": "{%- macro week_of_year(date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{{ adapter.dispatch('week_of_year', 'dbt_date') (dt) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_date.default__week_of_year"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.171505}, "macro.dbt_date.default__week_of_year": {"unique_id": "macro.dbt_date.default__week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_of_year.sql", "original_file_path": "macros/calendar_date/week_of_year.sql", "name": "default__week_of_year", "macro_sql": "{%- macro default__week_of_year(date) -%}\ncast({{ dbt_date.date_part('week', date) }} as {{ dbt_utils.type_int() }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_utils.type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1717062}, "macro.dbt_date.postgres__week_of_year": {"unique_id": "macro.dbt_date.postgres__week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_of_year.sql", "original_file_path": "macros/calendar_date/week_of_year.sql", "name": "postgres__week_of_year", "macro_sql": "\n\n{%- macro postgres__week_of_year(date) -%}\n{# postgresql 'week' returns isoweek. Use to_char instead.\n WW = the first week starts on the first day of the year #}\ncast(to_char({{ date }}, 'WW') as {{ dbt_utils.type_int() }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.171861}, "macro.dbt_date.convert_timezone": {"unique_id": "macro.dbt_date.convert_timezone", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/convert_timezone.sql", "original_file_path": "macros/calendar_date/convert_timezone.sql", "name": "convert_timezone", "macro_sql": "{%- macro convert_timezone(column, target_tz=None, source_tz=None) -%}\n{%- set source_tz = \"UTC\" if not source_tz else source_tz -%}\n{%- set target_tz = var(\"dbt_date:time_zone\") if not target_tz else target_tz -%}\n{{ adapter.dispatch('convert_timezone', 'dbt_date') (column, target_tz, source_tz) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.bigquery__convert_timezone"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.173192}, "macro.dbt_date.default__convert_timezone": {"unique_id": "macro.dbt_date.default__convert_timezone", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/convert_timezone.sql", "original_file_path": "macros/calendar_date/convert_timezone.sql", "name": "default__convert_timezone", "macro_sql": "{% macro default__convert_timezone(column, target_tz, source_tz) -%}\n{%- if not source_tz -%}\ncast(convert_timezone('{{ target_tz }}', {{ column }}) as {{ dbt_utils.type_timestamp() }})\n{%- else -%}\ncast(convert_timezone('{{ source_tz }}', '{{ target_tz }}', {{ column }}) as {{ dbt_utils.type_timestamp() }})\n{%- endif -%}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.173558}, "macro.dbt_date.bigquery__convert_timezone": {"unique_id": "macro.dbt_date.bigquery__convert_timezone", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/convert_timezone.sql", "original_file_path": "macros/calendar_date/convert_timezone.sql", "name": "bigquery__convert_timezone", "macro_sql": "{%- macro bigquery__convert_timezone(column, target_tz, source_tz=None) -%}\ntimestamp(datetime({{ column }}, '{{ target_tz}}'))\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.173723}, "macro.dbt_date.spark__convert_timezone": {"unique_id": "macro.dbt_date.spark__convert_timezone", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/convert_timezone.sql", "original_file_path": "macros/calendar_date/convert_timezone.sql", "name": "spark__convert_timezone", "macro_sql": "{%- macro spark__convert_timezone(column, target_tz, source_tz) -%}\nfrom_utc_timestamp(\n to_utc_timestamp({{ column }}, '{{ source_tz }}'),\n '{{ target_tz }}'\n )\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.173894}, "macro.dbt_date.postgres__convert_timezone": {"unique_id": "macro.dbt_date.postgres__convert_timezone", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/convert_timezone.sql", "original_file_path": "macros/calendar_date/convert_timezone.sql", "name": "postgres__convert_timezone", "macro_sql": "{% macro postgres__convert_timezone(column, target_tz, source_tz) -%}\n{%- if source_tz -%}\ncast({{ column }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' as {{ dbt_utils.type_timestamp() }})\n{%- else -%}\ncast({{ column }} at time zone '{{ target_tz }}' as {{ dbt_utils.type_timestamp() }})\n{%- endif -%}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.174305}, "macro.dbt_date.redshift__convert_timezone": {"unique_id": "macro.dbt_date.redshift__convert_timezone", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/convert_timezone.sql", "original_file_path": "macros/calendar_date/convert_timezone.sql", "name": "redshift__convert_timezone", "macro_sql": "{%- macro redshift__convert_timezone(column, target_tz, source_tz) -%}\n{{ return(dbt_date.default__convert_timezone(column, target_tz, source_tz)) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.default__convert_timezone"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.174508}, "macro.dbt_date.n_months_away": {"unique_id": "macro.dbt_date.n_months_away", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/n_months_away.sql", "original_file_path": "macros/calendar_date/n_months_away.sql", "name": "n_months_away", "macro_sql": "{%- macro n_months_away(n, tz=None) -%}\n{%- set n = n|int -%}\n{{ dbt_utils.date_trunc('month', \n dbt_utils.dateadd('month', n, \n dbt_date.today(tz)\n )\n ) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc", "macro.dbt_utils.dateadd", "macro.dbt_date.today"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.174968}, "macro.dbt_date.iso_week_of_year": {"unique_id": "macro.dbt_date.iso_week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_of_year.sql", "original_file_path": "macros/calendar_date/iso_week_of_year.sql", "name": "iso_week_of_year", "macro_sql": "{%- macro iso_week_of_year(date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{{ adapter.dispatch('iso_week_of_year', 'dbt_date') (dt) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_date.default__iso_week_of_year"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.175618}, "macro.dbt_date._iso_week_of_year": {"unique_id": "macro.dbt_date._iso_week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_of_year.sql", "original_file_path": "macros/calendar_date/iso_week_of_year.sql", "name": "_iso_week_of_year", "macro_sql": "{%- macro _iso_week_of_year(date, week_type) -%}\ncast({{ dbt_date.date_part(week_type, date) }} as {{ dbt_utils.type_int() }})\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_utils.type_int"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.175831}, "macro.dbt_date.default__iso_week_of_year": {"unique_id": "macro.dbt_date.default__iso_week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_of_year.sql", "original_file_path": "macros/calendar_date/iso_week_of_year.sql", "name": "default__iso_week_of_year", "macro_sql": "\n\n{%- macro default__iso_week_of_year(date) -%}\n{{ dbt_date._iso_week_of_year(date, 'isoweek') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_of_year"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.17598}, "macro.dbt_date.snowflake__iso_week_of_year": {"unique_id": "macro.dbt_date.snowflake__iso_week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_of_year.sql", "original_file_path": "macros/calendar_date/iso_week_of_year.sql", "name": "snowflake__iso_week_of_year", "macro_sql": "\n\n{%- macro snowflake__iso_week_of_year(date) -%}\n{{ dbt_date._iso_week_of_year(date, 'weekiso') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_of_year"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.176127}, "macro.dbt_date.postgres__iso_week_of_year": {"unique_id": "macro.dbt_date.postgres__iso_week_of_year", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/iso_week_of_year.sql", "original_file_path": "macros/calendar_date/iso_week_of_year.sql", "name": "postgres__iso_week_of_year", "macro_sql": "\n\n{%- macro postgres__iso_week_of_year(date) -%}\n-- postgresql week is isoweek, the first week of a year containing January 4 of that year.\n{{ dbt_date._iso_week_of_year(date, 'week') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date._iso_week_of_year"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1762838}, "macro.dbt_date.week_end": {"unique_id": "macro.dbt_date.week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_end.sql", "original_file_path": "macros/calendar_date/week_end.sql", "name": "week_end", "macro_sql": "{%- macro week_end(date=None, tz=None) -%}\n{%-set dt = date if date else dbt_date.today(tz) -%}\n{{ adapter.dispatch('week_end', 'dbt_date') (dt) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.today", "macro.dbt_date.default__week_end"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.176855}, "macro.dbt_date.default__week_end": {"unique_id": "macro.dbt_date.default__week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_end.sql", "original_file_path": "macros/calendar_date/week_end.sql", "name": "default__week_end", "macro_sql": "{%- macro default__week_end(date) -%}\n{{ dbt_utils.last_day(date, 'week') }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.last_day"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.177001}, "macro.dbt_date.snowflake__week_end": {"unique_id": "macro.dbt_date.snowflake__week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_end.sql", "original_file_path": "macros/calendar_date/week_end.sql", "name": "snowflake__week_end", "macro_sql": "\n\n{%- macro snowflake__week_end(date) -%}\n{%- set dt = dbt_date.week_start(date) -%}\n{{ dbt_date.n_days_away(6, dt) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.week_start", "macro.dbt_date.n_days_away"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.177213}, "macro.dbt_date.postgres__week_end": {"unique_id": "macro.dbt_date.postgres__week_end", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/week_end.sql", "original_file_path": "macros/calendar_date/week_end.sql", "name": "postgres__week_end", "macro_sql": "\n\n{%- macro postgres__week_end(date) -%}\n{%- set dt = dbt_date.week_start(date) -%}\n{{ dbt_date.n_days_away(6, dt) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.week_start", "macro.dbt_date.n_days_away"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.17743}, "macro.dbt_date.next_month_number": {"unique_id": "macro.dbt_date.next_month_number", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/next_month_number.sql", "original_file_path": "macros/calendar_date/next_month_number.sql", "name": "next_month_number", "macro_sql": "{%- macro next_month_number(tz=None) -%}\n{{ dbt_date.date_part('month', dbt_date.next_month(1, tz)) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_date.next_month"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1777558}, "macro.dbt_date.last_month_number": {"unique_id": "macro.dbt_date.last_month_number", "package_name": "dbt_date", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_date", "path": "macros/calendar_date/last_month_number.sql", "original_file_path": "macros/calendar_date/last_month_number.sql", "name": "last_month_number", "macro_sql": "{%- macro last_month_number(tz=None) -%}\n{{ dbt_date.date_part('month', dbt_date.last_month(1, tz)) }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_date.date_part", "macro.dbt_date.last_month"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.178078}, "macro.dbt_expectations.type_timestamp": {"unique_id": "macro.dbt_expectations.type_timestamp", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "type_timestamp", "macro_sql": "\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt_expectations')()) }}\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.178551}, "macro.dbt_expectations.default__type_timestamp": {"unique_id": "macro.dbt_expectations.default__type_timestamp", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() -%}\n timestamp\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1786299}, "macro.dbt_expectations.snowflake__type_timestamp": {"unique_id": "macro.dbt_expectations.snowflake__type_timestamp", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "snowflake__type_timestamp", "macro_sql": "{% macro snowflake__type_timestamp() -%}\n timestamp_ntz\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.178702}, "macro.dbt_expectations.postgres__type_timestamp": {"unique_id": "macro.dbt_expectations.postgres__type_timestamp", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "postgres__type_timestamp", "macro_sql": "{% macro postgres__type_timestamp() -%}\n timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.178776}, "macro.dbt_expectations.type_datetime": {"unique_id": "macro.dbt_expectations.type_datetime", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "type_datetime", "macro_sql": "{% macro type_datetime() -%}\n {{ return(adapter.dispatch('type_datetime', 'dbt_expectations')()) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__type_datetime"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.178943}, "macro.dbt_expectations.default__type_datetime": {"unique_id": "macro.dbt_expectations.default__type_datetime", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "default__type_datetime", "macro_sql": "{% macro default__type_datetime() -%}\n datetime\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.179017}, "macro.dbt_expectations.snowflake__type_datetime": {"unique_id": "macro.dbt_expectations.snowflake__type_datetime", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "snowflake__type_datetime", "macro_sql": "{% macro snowflake__type_datetime() -%}\n timestamp_ntz\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.179093}, "macro.dbt_expectations.postgres__type_datetime": {"unique_id": "macro.dbt_expectations.postgres__type_datetime", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/utils/datatypes.sql", "original_file_path": "macros/utils/datatypes.sql", "name": "postgres__type_datetime", "macro_sql": "{% macro postgres__type_datetime() -%}\n timestamp without time zone\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.179343}, "macro.dbt_expectations.regexp_instr": {"unique_id": "macro.dbt_expectations.regexp_instr", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/regex/regexp_instr.sql", "original_file_path": "macros/regex/regexp_instr.sql", "name": "regexp_instr", "macro_sql": "{% macro regexp_instr(source_value, regexp, position=1, occurrence=1) %}\n\n {{ adapter.dispatch('regexp_instr', 'dbt_expectations')(\n source_value, regexp, position, occurrence\n ) }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__regexp_instr"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.180049}, "macro.dbt_expectations.default__regexp_instr": {"unique_id": "macro.dbt_expectations.default__regexp_instr", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/regex/regexp_instr.sql", "original_file_path": "macros/regex/regexp_instr.sql", "name": "default__regexp_instr", "macro_sql": "{% macro default__regexp_instr(source_value, regexp, position, occurrence) %}\nregexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1802518}, "macro.dbt_expectations.redshift__regexp_instr": {"unique_id": "macro.dbt_expectations.redshift__regexp_instr", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/regex/regexp_instr.sql", "original_file_path": "macros/regex/regexp_instr.sql", "name": "redshift__regexp_instr", "macro_sql": "{% macro redshift__regexp_instr(source_value, regexp, position, occurrence) %}\nregexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.180453}, "macro.dbt_expectations.postgres__regexp_instr": {"unique_id": "macro.dbt_expectations.postgres__regexp_instr", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/regex/regexp_instr.sql", "original_file_path": "macros/regex/regexp_instr.sql", "name": "postgres__regexp_instr", "macro_sql": "{% macro postgres__regexp_instr(source_value, regexp, position, occurrence) %}\narray_length((select regexp_matches({{ source_value }}, '{{ regexp }}')), 1)\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.180613}, "macro.dbt_expectations.spark__regexp_instr": {"unique_id": "macro.dbt_expectations.spark__regexp_instr", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/regex/regexp_instr.sql", "original_file_path": "macros/regex/regexp_instr.sql", "name": "spark__regexp_instr", "macro_sql": "{% macro spark__regexp_instr(source_value, regexp, position, occurrence) %}\ncase when {{ source_value }} rlike '{{ regexp }}' then 1 else 0 end\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.180769}, "macro.dbt_expectations.log_natural": {"unique_id": "macro.dbt_expectations.log_natural", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/log_natural.sql", "original_file_path": "macros/math/log_natural.sql", "name": "log_natural", "macro_sql": "{% macro log_natural(x) -%}\n {{ adapter.dispatch('log_natural', 'dbt_expectations') (x) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.bigquery__log_natural"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1811478}, "macro.dbt_expectations.default__log_natural": {"unique_id": "macro.dbt_expectations.default__log_natural", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/log_natural.sql", "original_file_path": "macros/math/log_natural.sql", "name": "default__log_natural", "macro_sql": "{% macro default__log_natural(x) %}\n\n ln({{ x }})\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1812558}, "macro.dbt_expectations.bigquery__log_natural": {"unique_id": "macro.dbt_expectations.bigquery__log_natural", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/log_natural.sql", "original_file_path": "macros/math/log_natural.sql", "name": "bigquery__log_natural", "macro_sql": "{% macro bigquery__log_natural(x) %}\n\n ln({{ x }})\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.181362}, "macro.dbt_expectations.snowflake__log_natural": {"unique_id": "macro.dbt_expectations.snowflake__log_natural", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/log_natural.sql", "original_file_path": "macros/math/log_natural.sql", "name": "snowflake__log_natural", "macro_sql": "{% macro snowflake__log_natural(x) %}\n\n ln({{ x }})\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.181466}, "macro.dbt_expectations.rand": {"unique_id": "macro.dbt_expectations.rand", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/rand.sql", "original_file_path": "macros/math/rand.sql", "name": "rand", "macro_sql": "{% macro rand() -%}\n {{ adapter.dispatch('rand', 'dbt_expectations') () }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.bigquery__rand"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1818671}, "macro.dbt_expectations.default__rand": {"unique_id": "macro.dbt_expectations.default__rand", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/rand.sql", "original_file_path": "macros/math/rand.sql", "name": "default__rand", "macro_sql": "{% macro default__rand() %}\n\n rand()\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.181942}, "macro.dbt_expectations.bigquery__rand": {"unique_id": "macro.dbt_expectations.bigquery__rand", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/rand.sql", "original_file_path": "macros/math/rand.sql", "name": "bigquery__rand", "macro_sql": "{% macro bigquery__rand() %}\n\n rand()\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.182017}, "macro.dbt_expectations.snowflake__rand": {"unique_id": "macro.dbt_expectations.snowflake__rand", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/rand.sql", "original_file_path": "macros/math/rand.sql", "name": "snowflake__rand", "macro_sql": "{% macro snowflake__rand(seed) %}\n\n uniform(0::float, 1::float, random())\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.182097}, "macro.dbt_expectations.postgres__rand": {"unique_id": "macro.dbt_expectations.postgres__rand", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/rand.sql", "original_file_path": "macros/math/rand.sql", "name": "postgres__rand", "macro_sql": "{% macro postgres__rand() %}\n\n random()\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.182173}, "macro.dbt_expectations.redshift__rand": {"unique_id": "macro.dbt_expectations.redshift__rand", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/rand.sql", "original_file_path": "macros/math/rand.sql", "name": "redshift__rand", "macro_sql": "{% macro redshift__rand() %}\n\n random()\n\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1822798}, "macro.dbt_expectations.median": {"unique_id": "macro.dbt_expectations.median", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/median.sql", "original_file_path": "macros/math/median.sql", "name": "median", "macro_sql": "{% macro median(field) %}\n{{ dbt_expectations.percentile_cont(field, 0.5) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.percentile_cont"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.182556}, "macro.dbt_expectations.percentile_cont": {"unique_id": "macro.dbt_expectations.percentile_cont", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/percentile_cont.sql", "original_file_path": "macros/math/percentile_cont.sql", "name": "percentile_cont", "macro_sql": "{% macro percentile_cont(field, quantile, partition=None) %}\n {{ adapter.dispatch('quantile', 'dbt_expectations') (field, quantile, partition) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.bigquery__quantile"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.183085}, "macro.dbt_expectations.default__quantile": {"unique_id": "macro.dbt_expectations.default__quantile", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/percentile_cont.sql", "original_file_path": "macros/math/percentile_cont.sql", "name": "default__quantile", "macro_sql": "{% macro default__quantile(field, quantile, partition) -%}\n percentile_cont({{ quantile }}) within group (order by {{ field }})\n {%- if partition %}over(partition by {{ partition }}){% endif -%}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.183307}, "macro.dbt_expectations.bigquery__quantile": {"unique_id": "macro.dbt_expectations.bigquery__quantile", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/math/percentile_cont.sql", "original_file_path": "macros/math/percentile_cont.sql", "name": "bigquery__quantile", "macro_sql": "{% macro bigquery__quantile(field, quantile, partition) %}\n percentile_cont({{ field }}, {{ quantile }})\n over({%- if partition %}partition by {{ partition }}{% endif -%})\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.183594}, "macro.dbt_expectations.test_expression_between": {"unique_id": "macro.dbt_expectations.test_expression_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/expression_between.sql", "original_file_path": "macros/schema_tests/_generalized/expression_between.sql", "name": "test_expression_between", "macro_sql": "{% test expression_between(model,\n expression,\n min_value=None,\n max_value=None,\n group_by_columns=None,\n row_condition=None,\n strictly=False\n ) %}\n\n {{ dbt_expectations.expression_between(model, expression, min_value, max_value, group_by_columns, row_condition, strictly) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1848981}, "macro.dbt_expectations.expression_between": {"unique_id": "macro.dbt_expectations.expression_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/expression_between.sql", "original_file_path": "macros/schema_tests/_generalized/expression_between.sql", "name": "expression_between", "macro_sql": "{% macro expression_between(model,\n expression,\n min_value,\n max_value,\n group_by_columns,\n row_condition,\n strictly\n ) %}\n\n{%- if min_value is none and max_value is none -%}\n{{ exceptions.raise_compiler_error(\n \"You have to provide either a min_value, max_value or both.\"\n) }}\n{%- endif -%}\n\n{%- set strict_operator = \"\" if strictly else \"=\" -%}\n\n{% set expression_min_max %}\n( 1=1\n{%- if min_value is not none %} and {{ expression | trim }} >{{ strict_operator }} {{ min_value }}{% endif %}\n{%- if max_value is not none %} and {{ expression | trim }} <{{ strict_operator }} {{ max_value }}{% endif %}\n)\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression_min_max,\n group_by_columns=group_by_columns,\n row_condition=row_condition)\n }}\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.185718}, "macro.dbt_expectations.test_expression_is_true": {"unique_id": "macro.dbt_expectations.test_expression_is_true", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/expression_is_true.sql", "original_file_path": "macros/schema_tests/_generalized/expression_is_true.sql", "name": "test_expression_is_true", "macro_sql": "{% test expression_is_true(model,\n expression,\n test_condition=\"= true\",\n group_by_columns=None,\n row_condition=None\n ) %}\n\n {{ dbt_expectations.expression_is_true(model, expression, test_condition, group_by_columns, row_condition) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1867929}, "macro.dbt_expectations.expression_is_true": {"unique_id": "macro.dbt_expectations.expression_is_true", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/expression_is_true.sql", "original_file_path": "macros/schema_tests/_generalized/expression_is_true.sql", "name": "expression_is_true", "macro_sql": "{% macro expression_is_true(model,\n expression,\n test_condition=\"= true\",\n group_by_columns=None,\n row_condition=None\n ) %}\n {{ adapter.dispatch('expression_is_true', 'dbt_expectations') (model, expression, test_condition, group_by_columns, row_condition) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.187117}, "macro.dbt_expectations.default__expression_is_true": {"unique_id": "macro.dbt_expectations.default__expression_is_true", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/expression_is_true.sql", "original_file_path": "macros/schema_tests/_generalized/expression_is_true.sql", "name": "default__expression_is_true", "macro_sql": "{% macro default__expression_is_true(model, expression, test_condition, group_by_columns, row_condition) -%}\nwith grouped_expression as (\n select\n {% if group_by_columns %}\n {% for group_by_column in group_by_columns -%}\n {{ group_by_column }} as col_{{ loop.index }},\n {% endfor -%}\n {% endif %}\n {{ dbt_expectations.truth_expression(expression) }}\n from {{ model }}\n {%- if row_condition %}\n where\n {{ row_condition }}\n {% endif %}\n {% if group_by_columns %}\n group by\n {% for group_by_column in group_by_columns -%}\n {{ group_by_column }}{% if not loop.last %},{% endif %}\n {% endfor %}\n {% endif %}\n\n),\nvalidation_errors as (\n\n select\n *\n from\n grouped_expression\n where\n not(expression {{ test_condition }})\n\n)\n\nselect *\nfrom validation_errors\n\n\n{% endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.truth_expression"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.187804}, "macro.dbt_expectations.get_select": {"unique_id": "macro.dbt_expectations.get_select", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/equal_expression.sql", "original_file_path": "macros/schema_tests/_generalized/equal_expression.sql", "name": "get_select", "macro_sql": "{% macro get_select(model, expression, row_condition, group_by) -%}\n {{ adapter.dispatch('get_select', 'dbt_expectations') (model, expression, row_condition, group_by) }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__get_select"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.193347}, "macro.dbt_expectations.default__get_select": {"unique_id": "macro.dbt_expectations.default__get_select", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/equal_expression.sql", "original_file_path": "macros/schema_tests/_generalized/equal_expression.sql", "name": "default__get_select", "macro_sql": "\n\n{%- macro default__get_select(model, expression, row_condition, group_by) %}\n select\n {% if group_by %}\n {% for g in group_by -%}\n {{ g }} as col_{{ loop.index }},\n {% endfor -%}\n {% endif %}\n {{ expression }} as expression\n from\n {{ model }}\n {%- if row_condition %}\n where\n {{ row_condition }}\n {% endif %}\n {% if group_by %}\n group by\n {% for g in group_by -%}\n {{ loop.index }}{% if not loop.last %},{% endif %}\n {% endfor %}\n {% endif %}\n{% endmacro -%}\n\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1939578}, "macro.dbt_expectations.test_equal_expression": {"unique_id": "macro.dbt_expectations.test_equal_expression", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/equal_expression.sql", "original_file_path": "macros/schema_tests/_generalized/equal_expression.sql", "name": "test_equal_expression", "macro_sql": "{% test equal_expression(model, expression,\n compare_model=None,\n compare_expression=None,\n group_by=None,\n compare_group_by=None,\n row_condition=None,\n compare_row_condition=None,\n tolerance=0.0,\n tolerance_percent=None\n ) -%}\n\n {{ adapter.dispatch('test_equal_expression', 'dbt_expectations') (\n model,\n expression,\n compare_model,\n compare_expression,\n group_by,\n compare_group_by,\n row_condition,\n compare_row_condition,\n tolerance,\n tolerance_percent) }}\n{%- endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_equal_expression"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1944818}, "macro.dbt_expectations.default__test_equal_expression": {"unique_id": "macro.dbt_expectations.default__test_equal_expression", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/equal_expression.sql", "original_file_path": "macros/schema_tests/_generalized/equal_expression.sql", "name": "default__test_equal_expression", "macro_sql": "\n\n{%- macro default__test_equal_expression(\n model,\n expression,\n compare_model,\n compare_expression,\n group_by,\n compare_group_by,\n row_condition,\n compare_row_condition,\n tolerance,\n tolerance_percent) -%}\n\n {%- set compare_model = model if not compare_model else compare_model -%}\n {%- set compare_expression = expression if not compare_expression else compare_expression -%}\n {%- set compare_row_condition = row_condition if not compare_row_condition else compare_row_condition -%}\n {%- set compare_group_by = group_by if not compare_group_by else compare_group_by -%}\n\n {%- set n_cols = (group_by|length) if group_by else 0 %}\n with a as (\n {{ dbt_expectations.get_select(model, expression, row_condition, group_by) }}\n ),\n b as (\n {{ dbt_expectations.get_select(compare_model, compare_expression, compare_row_condition, compare_group_by) }}\n ),\n final as (\n\n select\n {% for i in range(1, n_cols + 1) -%}\n coalesce(a.col_{{ i }}, b.col_{{ i }}) as col_{{ i }},\n {% endfor %}\n a.expression,\n b.expression as compare_expression,\n abs(coalesce(a.expression, 0) - coalesce(b.expression, 0)) as expression_difference,\n abs(coalesce(a.expression, 0) - coalesce(b.expression, 0))/\n nullif(a.expression * 1.0, 0) as expression_difference_percent\n from\n {% if n_cols > 0 %}\n a\n full outer join\n b on\n {% for i in range(1, n_cols + 1) -%}\n a.col_{{ i }} = b.col_{{ i }} {% if not loop.last %}and{% endif %}\n {% endfor -%}\n {% else %}\n a cross join b\n {% endif %}\n )\n -- DEBUG:\n -- select * from final\n select\n *\n from final\n where\n {% if tolerance_percent %}\n expression_difference_percent > {{ tolerance_percent }}\n {% else %}\n expression_difference > {{ tolerance }}\n {% endif %}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.get_select"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1958961}, "macro.dbt_expectations.truth_expression": {"unique_id": "macro.dbt_expectations.truth_expression", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/_truth_expression.sql", "original_file_path": "macros/schema_tests/_generalized/_truth_expression.sql", "name": "truth_expression", "macro_sql": "{% macro truth_expression(expression) %}\n {{ adapter.dispatch('truth_expression', 'dbt_expectations') (expression) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__truth_expression"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.196249}, "macro.dbt_expectations.default__truth_expression": {"unique_id": "macro.dbt_expectations.default__truth_expression", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/_generalized/_truth_expression.sql", "original_file_path": "macros/schema_tests/_generalized/_truth_expression.sql", "name": "default__truth_expression", "macro_sql": "{% macro default__truth_expression(expression) %}\n {{ expression }} as expression\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.196359}, "macro.dbt_expectations.test_expect_column_values_to_match_like_pattern": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_match_like_pattern", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern.sql", "name": "test_expect_column_values_to_match_like_pattern", "macro_sql": "{% test expect_column_values_to_match_like_pattern(model, column_name,\n like_pattern,\n row_condition=None\n ) %}\n\n{% set expression = dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=True) %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_like_pattern_expression", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1970098}, "macro.dbt_expectations.test_expect_column_values_to_match_like_pattern_list": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_match_like_pattern_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern_list.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern_list.sql", "name": "test_expect_column_values_to_match_like_pattern_list", "macro_sql": "{% test expect_column_values_to_match_like_pattern_list(model, column_name,\n like_pattern_list,\n match_on=\"any\",\n row_condition=None\n ) %}\n\n{% set expression %}\n {% for like_pattern in like_pattern_list %}\n {{ dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=True) }}\n {%- if not loop.last %}\n {{ \" and \" if match_on == \"all\" else \" or \"}}\n {% endif -%}\n {% endfor %}\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_like_pattern_expression", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.198176}, "macro.dbt_expectations.test_expect_column_values_to_match_regex": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_match_regex", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_match_regex.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_match_regex.sql", "name": "test_expect_column_values_to_match_regex", "macro_sql": "{% test expect_column_values_to_match_regex(model, column_name,\n regex,\n row_condition=None\n ) %}\n\n{% set expression %}\n{{ dbt_expectations.regexp_instr(column_name, regex) }} > 0\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.regexp_instr", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.1989}, "macro.dbt_expectations.test_expect_column_value_lengths_to_equal": {"unique_id": "macro.dbt_expectations.test_expect_column_value_lengths_to_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_value_lengths_to_equal.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_value_lengths_to_equal.sql", "name": "test_expect_column_value_lengths_to_equal", "macro_sql": "{% test expect_column_value_lengths_to_equal(model, column_name,\n value,\n row_condition=None\n ) %}\n\n{% set expression = dbt_utils.length(column_name) ~ \" = \" ~ value %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.length", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.199532}, "macro.dbt_expectations.test_expect_column_value_lengths_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_value_lengths_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_value_lengths_to_be_between.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_value_lengths_to_be_between.sql", "name": "test_expect_column_value_lengths_to_be_between", "macro_sql": "{% test expect_column_value_lengths_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\n{{ dbt_utils.length(column_name) }}\n{% endset %}\n\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=None,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.length", "macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.200449}, "macro.dbt_expectations.test_expect_column_values_to_not_match_regex": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_not_match_regex", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_regex.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_regex.sql", "name": "test_expect_column_values_to_not_match_regex", "macro_sql": "{% test expect_column_values_to_not_match_regex(model, column_name,\n regex,\n row_condition=None\n ) %}\n\n{% set expression %}\n{{ dbt_expectations.regexp_instr(column_name, regex) }} = 0\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.regexp_instr", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.201161}, "macro.dbt_expectations.test_expect_column_values_to_not_match_regex_list": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_not_match_regex_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_regex_list.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_regex_list.sql", "name": "test_expect_column_values_to_not_match_regex_list", "macro_sql": "{% test expect_column_values_to_not_match_regex_list(model, column_name,\n regex_list,\n match_on=\"any\",\n row_condition=None\n ) %}\n\n{% set expression %}\n{% for regex in regex_list %}\n{{ dbt_expectations.regexp_instr(column_name, regex) }} = 0\n{%- if not loop.last %}\n{{ \" and \" if match_on == \"all\" else \" or \"}}\n{% endif -%}\n{% endfor %}\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.regexp_instr", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2023628}, "macro.dbt_expectations.test_expect_column_values_to_match_regex_list": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_match_regex_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_match_regex_list.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_match_regex_list.sql", "name": "test_expect_column_values_to_match_regex_list", "macro_sql": "{% test expect_column_values_to_match_regex_list(model, column_name,\n regex_list,\n match_on=\"any\",\n row_condition=None\n ) %}\n\n{% set expression %}\n {% for regex in regex_list %}\n {{ dbt_expectations.regexp_instr(column_name, regex) }} > 0\n {%- if not loop.last %}\n {{ \" and \" if match_on == \"all\" else \" or \"}}\n {% endif -%}\n {% endfor %}\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.regexp_instr", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.203506}, "macro.dbt_expectations.test_expect_column_values_to_not_match_like_pattern_list": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_not_match_like_pattern_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_like_pattern_list.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_like_pattern_list.sql", "name": "test_expect_column_values_to_not_match_like_pattern_list", "macro_sql": "{% test expect_column_values_to_not_match_like_pattern_list(model, column_name,\n like_pattern_list,\n match_on=\"any\",\n row_condition=None\n ) %}\n\n{% set expression %}\n {% for like_pattern in like_pattern_list %}\n {{ dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=False) }}\n {%- if not loop.last %}\n {{ \" and \" if match_on == \"all\" else \" or \"}}\n {% endif -%}\n {% endfor %}\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_like_pattern_expression", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.204671}, "macro.dbt_expectations._get_like_pattern_expression": {"unique_id": "macro.dbt_expectations._get_like_pattern_expression", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/_get_like_pattern_expression.sql", "original_file_path": "macros/schema_tests/string_matching/_get_like_pattern_expression.sql", "name": "_get_like_pattern_expression", "macro_sql": "{% macro _get_like_pattern_expression(column_name, like_pattern, positive) %}\n{{ column_name }} {{ \"not\" if not positive else \"\" }} like '{{ like_pattern }}'\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.205031}, "macro.dbt_expectations.test_expect_column_values_to_not_match_like_pattern": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_not_match_like_pattern", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_like_pattern.sql", "original_file_path": "macros/schema_tests/string_matching/expect_column_values_to_not_match_like_pattern.sql", "name": "test_expect_column_values_to_not_match_like_pattern", "macro_sql": "{% test expect_column_values_to_not_match_like_pattern(model, column_name,\n like_pattern,\n row_condition=None\n ) %}\n\n{% set expression = dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=False) %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_like_pattern_expression", "macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2056818}, "macro.dbt_expectations.test_expect_row_values_to_have_recent_data": {"unique_id": "macro.dbt_expectations.test_expect_row_values_to_have_recent_data", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_row_values_to_have_recent_data.sql", "original_file_path": "macros/schema_tests/table_shape/expect_row_values_to_have_recent_data.sql", "name": "test_expect_row_values_to_have_recent_data", "macro_sql": "{% test expect_row_values_to_have_recent_data(model,\n column_name,\n datepart,\n interval,\n row_condition=None) %}\n\n {{ adapter.dispatch('test_expect_row_values_to_have_recent_data', 'dbt_expectations') (model,\n column_name,\n datepart,\n interval,\n row_condition) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_row_values_to_have_recent_data"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.206926}, "macro.dbt_expectations.default__test_expect_row_values_to_have_recent_data": {"unique_id": "macro.dbt_expectations.default__test_expect_row_values_to_have_recent_data", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_row_values_to_have_recent_data.sql", "original_file_path": "macros/schema_tests/table_shape/expect_row_values_to_have_recent_data.sql", "name": "default__test_expect_row_values_to_have_recent_data", "macro_sql": "{% macro default__test_expect_row_values_to_have_recent_data(model, column_name, datepart, interval, row_condition) %}\n{%- set default_start_date = '1970-01-01' -%}\nwith max_recency as (\n\n select max(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})) as max_timestamp\n from\n {{ model }}\n where\n cast({{ column_name }} as {{ dbt_utils.type_timestamp() }}) <= {{ dbt_date.now() }}\n {% if row_condition %}\n and {{ row_condition }}\n {% endif %}\n)\nselect\n *\nfrom\n max_recency\nwhere\n -- if the row_condition excludes all row, we need to compare against a default date\n -- to avoid false negatives\n coalesce(max_timestamp, cast('{{ default_start_date }}' as {{ dbt_utils.type_timestamp() }}))\n <\n cast({{ dbt_utils.dateadd(datepart, interval * -1, dbt_date.now()) }} as {{ dbt_utils.type_timestamp() }})\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_timestamp", "macro.dbt_date.now", "macro.dbt_utils.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.207608}, "macro.dbt_expectations.test_expect_table_columns_to_contain_set": {"unique_id": "macro.dbt_expectations.test_expect_table_columns_to_contain_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql", "name": "test_expect_table_columns_to_contain_set", "macro_sql": "{%- test expect_table_columns_to_contain_set(model, column_list, transform=\"upper\") -%}\n{%- if execute -%}\n {%- set column_list = column_list | map(transform) | list -%}\n {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%}\n {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%}\n with relation_columns as (\n\n {% for col_name in relation_column_names %}\n select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n ),\n input_columns as (\n\n {% for col_name in column_list %}\n select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n )\n select *\n from\n input_columns i\n left join\n relation_columns r on r.relation_column = i.input_column\n where\n -- catch any column in input list that is not in the list of table columns\n r.relation_column is null\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_column_list", "macro.dbt_expectations._list_intersect", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2089431}, "macro.dbt_expectations.test_expect_table_row_count_to_equal_other_table": {"unique_id": "macro.dbt_expectations.test_expect_table_row_count_to_equal_other_table", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal_other_table.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal_other_table.sql", "name": "test_expect_table_row_count_to_equal_other_table", "macro_sql": "{%- test expect_table_row_count_to_equal_other_table(model, compare_model, factor=1, row_condition=None, compare_row_condition=None) -%}\n{{ dbt_expectations.test_equal_expression(model, \"count(*)\",\n compare_model=compare_model,\n compare_expression=\"count(*) * \" + factor|string,\n row_condition=row_condition,\n compare_row_condition=compare_row_condition\n) }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.test_equal_expression"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.209454}, "macro.dbt_expectations.test_expect_table_columns_to_not_contain_set": {"unique_id": "macro.dbt_expectations.test_expect_table_columns_to_not_contain_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_columns_to_not_contain_set.sql", "name": "test_expect_table_columns_to_not_contain_set", "macro_sql": "{%- test expect_table_columns_to_not_contain_set(model, column_list, transform=\"upper\") -%}\n{%- if execute -%}\n {%- set column_list = column_list | map(transform) | list -%}\n {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%}\n {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%}\n with relation_columns as (\n\n {% for col_name in relation_column_names %}\n select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n ),\n input_columns as (\n\n {% for col_name in column_list %}\n select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n )\n -- catch any column in input list that is in the list of table columns\n select *\n from\n input_columns i\n inner join\n relation_columns r on r.relation_column = i.input_column\n\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_column_list", "macro.dbt_expectations._list_intersect", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.210758}, "macro.dbt_expectations.test_expect_grouped_row_values_to_have_recent_data": {"unique_id": "macro.dbt_expectations.test_expect_grouped_row_values_to_have_recent_data", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql", "original_file_path": "macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql", "name": "test_expect_grouped_row_values_to_have_recent_data", "macro_sql": "{% test expect_grouped_row_values_to_have_recent_data(model,\n group_by,\n timestamp_column,\n datepart,\n interval,\n row_condition=None) %}\n\n {{ adapter.dispatch('test_expect_grouped_row_values_to_have_recent_data', 'dbt_expectations') (model,\n group_by,\n timestamp_column,\n datepart,\n interval,\n row_condition) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_grouped_row_values_to_have_recent_data"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.212671}, "macro.dbt_expectations.default__test_expect_grouped_row_values_to_have_recent_data": {"unique_id": "macro.dbt_expectations.default__test_expect_grouped_row_values_to_have_recent_data", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql", "original_file_path": "macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql", "name": "default__test_expect_grouped_row_values_to_have_recent_data", "macro_sql": "{% macro default__test_expect_grouped_row_values_to_have_recent_data(model,\n group_by,\n timestamp_column,\n datepart,\n interval,\n row_condition) %}\nwith latest_grouped_timestamps as (\n\n select\n {%- for g in group_by %}\n {{ g }},\n {%- endfor %}\n max(1) as join_key,\n max({{ timestamp_column }}) as latest_timestamp_column\n from\n {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n {{ dbt_utils.group_by(group_by | length )}}\n\n),\ntotal_row_counts as (\n\n select\n max(1) as join_key,\n count(*) as row_count\n from\n latest_grouped_timestamps\n\n),\noutdated_grouped_timestamps as (\n\n select *\n from\n latest_grouped_timestamps\n where\n latest_timestamp_column < {{ dbt_utils.dateadd(datepart, interval * -1, dbt_date.now()) }}\n\n),\nvalidation_errors as (\n\n select\n r.row_count,\n t.*\n from\n total_row_counts r\n left join\n outdated_grouped_timestamps t\n on r.join_key = t.join_key\n where\n -- fail if either no rows were returned due to row_condition,\n -- or the recency test returned failed rows\n r.row_count = 0\n or\n t.join_key is not null\n\n)\nselect * from validation_errors\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.group_by", "macro.dbt_utils.dateadd", "macro.dbt_date.now"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.213259}, "macro.dbt_expectations.test_expect_column_to_exist": {"unique_id": "macro.dbt_expectations.test_expect_column_to_exist", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_column_to_exist.sql", "original_file_path": "macros/schema_tests/table_shape/expect_column_to_exist.sql", "name": "test_expect_column_to_exist", "macro_sql": "{%- test expect_column_to_exist(model, column_name, column_index=None, transform=\"upper\") -%}\n{%- if execute -%}\n\n {%- set column_name = column_name | map(transform) | join -%}\n {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%}\n\n {%- set matching_column_index = relation_column_names.index(column_name) if column_name in relation_column_names else -1 %}\n\n {%- if column_index -%}\n\n {%- set column_index_0 = column_index - 1 if column_index > 0 else 0 -%}\n\n {%- set column_index_matches = true if matching_column_index == column_index_0 else false %}\n\n {%- else -%}\n\n {%- set column_index_matches = true -%}\n\n {%- endif %}\n\n with test_data as (\n\n select\n cast('{{ column_name }}' as {{ dbt_utils.type_string() }}) as column_name,\n {{ matching_column_index }} as matching_column_index,\n {{ column_index_matches }} as column_index_matches\n\n )\n select *\n from test_data\n where\n not(matching_column_index >= 0 and column_index_matches)\n\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_column_list", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.214606}, "macro.dbt_expectations.test_expect_table_row_count_to_equal": {"unique_id": "macro.dbt_expectations.test_expect_table_row_count_to_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal.sql", "name": "test_expect_table_row_count_to_equal", "macro_sql": "{%- test expect_table_row_count_to_equal(model,\n value,\n group_by=None,\n row_condition=None\n ) -%}\n {{ adapter.dispatch('test_expect_table_row_count_to_equal',\n 'dbt_expectations') (model,\n value,\n group_by,\n row_condition\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_table_row_count_to_equal"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.215714}, "macro.dbt_expectations.default__test_expect_table_row_count_to_equal": {"unique_id": "macro.dbt_expectations.default__test_expect_table_row_count_to_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal.sql", "name": "default__test_expect_table_row_count_to_equal", "macro_sql": "\n\n\n\n{%- macro default__test_expect_table_row_count_to_equal(model,\n value,\n group_by,\n row_condition\n ) -%}\n{% set expression %}\ncount(*) = {{ value }}\n{% endset %}\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=group_by,\n row_condition=row_condition)\n }}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.216021}, "macro.dbt_expectations.test_expect_table_row_count_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_table_row_count_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_row_count_to_be_between.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_row_count_to_be_between.sql", "name": "test_expect_table_row_count_to_be_between", "macro_sql": "{%- test expect_table_row_count_to_be_between(model,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) -%}\n{% set expression %}\ncount(*)\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.216856}, "macro.dbt_expectations.test_expect_table_row_count_to_equal_other_table_times_factor": {"unique_id": "macro.dbt_expectations.test_expect_table_row_count_to_equal_other_table_times_factor", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal_other_table_times_factor.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_row_count_to_equal_other_table_times_factor.sql", "name": "test_expect_table_row_count_to_equal_other_table_times_factor", "macro_sql": "{%- test expect_table_row_count_to_equal_other_table_times_factor(model, compare_model, factor, row_condition=None, compare_row_condition=None) -%}\n{{ dbt_expectations.test_expect_table_row_count_to_equal_other_table(model, compare_model,\n factor=factor,\n row_condition=row_condition,\n compare_row_condition=compare_row_condition\n) }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.test_expect_table_row_count_to_equal_other_table"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2172768}, "macro.dbt_expectations.test_expect_table_columns_to_match_set": {"unique_id": "macro.dbt_expectations.test_expect_table_columns_to_match_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql", "name": "test_expect_table_columns_to_match_set", "macro_sql": "{%- test expect_table_columns_to_match_set(model, column_list, transform=\"upper\") -%}\n{%- if execute -%}\n {%- set column_list = column_list | map(transform) | list -%}\n {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%}\n {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%}\n with relation_columns as (\n\n {% for col_name in relation_column_names %}\n select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n ),\n input_columns as (\n\n {% for col_name in column_list %}\n select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n )\n select *\n from\n relation_columns r\n full outer join\n input_columns i on r.relation_column = i.input_column\n where\n -- catch any column in input list that is not in the list of table columns\n -- or any table column that is not in the input list\n r.relation_column is null or\n i.input_column is null\n\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_column_list", "macro.dbt_expectations._list_intersect", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.218722}, "macro.dbt_expectations._get_column_list": {"unique_id": "macro.dbt_expectations._get_column_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/_get_column_list.sql", "original_file_path": "macros/schema_tests/table_shape/_get_column_list.sql", "name": "_get_column_list", "macro_sql": "{%- macro _get_column_list(model, transform=\"upper\") -%}\n{%- set relation_columns = adapter.get_columns_in_relation(model) -%}\n{%- set relation_column_names = relation_columns | map(attribute=\"name\") | map(transform) | list -%}\n{%- do return(relation_column_names) -%}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.219214}, "macro.dbt_expectations.test_expect_table_columns_to_match_ordered_list": {"unique_id": "macro.dbt_expectations.test_expect_table_columns_to_match_ordered_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql", "name": "test_expect_table_columns_to_match_ordered_list", "macro_sql": "{%- test expect_table_columns_to_match_ordered_list(model, column_list, transform=\"upper\") -%}\n{%- if execute -%}\n {%- set column_list = column_list | map(transform) | list -%}\n {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%}\n {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%}\n with relation_columns as (\n\n {% for col_name in relation_column_names %}\n select\n {{ loop.index }} as relation_column_idx,\n cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n ),\n input_columns as (\n\n {% for col_name in column_list %}\n select\n {{ loop.index }} as input_column_idx,\n cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n )\n select *\n from\n relation_columns r\n full outer join\n input_columns i on r.relation_column = i.input_column and r.relation_column_idx = i.input_column_idx\n where\n -- catch any column in input list that is not in the sequence of table columns\n -- or any table column that is not in the input sequence\n r.relation_column is null or\n i.input_column is null\n\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations._get_column_list", "macro.dbt_expectations._list_intersect", "macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.220787}, "macro.dbt_expectations._list_intersect": {"unique_id": "macro.dbt_expectations._list_intersect", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/_list_intersect.sql", "original_file_path": "macros/schema_tests/table_shape/_list_intersect.sql", "name": "_list_intersect", "macro_sql": "{%- macro _list_intersect(list1, list2) -%}\n{%- set matching_items = [] -%}\n{%- for itm in list1 -%}\n {%- if itm in list2 -%}\n {%- do matching_items.append(itm) -%}\n {%- endif -%}\n{%- endfor -%}\n{%- do return(matching_items) -%}\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.221306}, "macro.dbt_expectations.test_expect_table_column_count_to_equal_other_table": {"unique_id": "macro.dbt_expectations.test_expect_table_column_count_to_equal_other_table", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql", "name": "test_expect_table_column_count_to_equal_other_table", "macro_sql": "{%- test expect_table_column_count_to_equal_other_table(model, compare_model) -%}\n{%- if execute -%}\n{%- set number_columns = (adapter.get_columns_in_relation(model) | length) -%}\n{%- set compare_number_columns = (adapter.get_columns_in_relation(compare_model) | length) -%}\nwith test_data as (\n\n select\n {{ number_columns }} as number_columns,\n {{ compare_number_columns }} as compare_number_columns\n\n)\nselect *\nfrom test_data\nwhere\n number_columns != compare_number_columns\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.221889}, "macro.dbt_expectations.test_expect_table_column_count_to_equal": {"unique_id": "macro.dbt_expectations.test_expect_table_column_count_to_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql", "name": "test_expect_table_column_count_to_equal", "macro_sql": "{%- test expect_table_column_count_to_equal(model, value) -%}\n{%- if execute -%}\n{%- set number_actual_columns = (adapter.get_columns_in_relation(model) | length) -%}\nwith test_data as (\n\n select\n {{ number_actual_columns }} as number_actual_columns,\n {{ value }} as value\n\n)\nselect *\nfrom test_data\nwhere\n number_actual_columns != value\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.222358}, "macro.dbt_expectations.test_expect_table_column_count_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_table_column_count_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/table_shape/expect_table_column_count_to_be_between.sql", "original_file_path": "macros/schema_tests/table_shape/expect_table_column_count_to_be_between.sql", "name": "test_expect_table_column_count_to_be_between", "macro_sql": "{%- test expect_table_column_count_to_be_between(model,\n min_value=None,\n max_value=None\n ) -%}\n{%- if min_value is none and max_value is none -%}\n{{ exceptions.raise_compiler_error(\n \"You have to provide either a min_value, max_value or both.\"\n) }}\n{%- endif -%}\n{%- if execute -%}\n{%- set number_actual_columns = (adapter.get_columns_in_relation(model) | length) -%}\n\n{%- set expression %}\n( 1=1\n{%- if min_value %} and number_actual_columns >= min_value{% endif %}\n{%- if max_value %} and number_actual_columns <= max_value{% endif %}\n)\n{% endset -%}\n\nwith test_data as (\n\n select\n {{ number_actual_columns }} as number_actual_columns,\n {{ min_value if min_value else 0 }} as min_value,\n {{ max_value if max_value else 0 }} as max_value\n\n)\nselect *\nfrom test_data\nwhere\n not {{ expression }}\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.223476}, "macro.dbt_expectations.test_expect_column_values_to_not_be_in_set": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_not_be_in_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql", "name": "test_expect_column_values_to_not_be_in_set", "macro_sql": "{% test expect_column_values_to_not_be_in_set(model, column_name,\n value_set,\n quote_values=True,\n row_condition=None\n ) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field\n\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nset_values as (\n\n {% for value in value_set -%}\n select\n {% if quote_values -%}\n cast('{{ value }}' as {{ dbt_utils.type_string() }})\n {%- else -%}\n {{ value }}\n {%- endif %} as value_field\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n),\nvalidation_errors as (\n -- values from the model that match the set\n select\n v.value_field\n from\n all_values v\n join\n set_values s on v.value_field = s.value_field\n\n)\n\nselect *\nfrom validation_errors\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2244968}, "macro.dbt_expectations.test_expect_column_values_to_be_in_set": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_in_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql", "name": "test_expect_column_values_to_be_in_set", "macro_sql": "{% test expect_column_values_to_be_in_set(model, column_name,\n value_set,\n quote_values=True,\n row_condition=None\n ) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field\n\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nset_values as (\n\n {% for value in value_set -%}\n select\n {% if quote_values -%}\n cast('{{ value }}' as {{ dbt_utils.type_string() }})\n {%- else -%}\n {{ value }}\n {%- endif %} as value_field\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n),\nvalidation_errors as (\n -- values from the model that are not in the set\n select\n v.value_field\n from\n all_values v\n left join\n set_values s on v.value_field = s.value_field\n where\n s.value_field is null\n\n)\n\nselect *\nfrom validation_errors\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.225535}, "macro.dbt_expectations.test_expect_column_values_to_be_increasing": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_increasing", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_increasing.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_increasing.sql", "name": "test_expect_column_values_to_be_increasing", "macro_sql": "{% test expect_column_values_to_be_increasing(model, column_name,\n sort_column=None,\n strictly=True,\n row_condition=None,\n group_by=None) %}\n\n{%- set sort_column = column_name if not sort_column else sort_column -%}\n{%- set operator = \">\" if strictly else \">=\" -%}\nwith all_values as (\n\n select\n {{ sort_column }} as sort_column,\n {%- if group_by -%}\n {{ group_by | join(\", \") }},\n {%- endif %}\n {{ column_name }} as value_field\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nadd_lag_values as (\n\n select\n sort_column,\n {%- if group_by -%}\n {{ group_by | join(\", \") }},\n {%- endif %}\n value_field,\n lag(value_field) over\n {%- if not group_by -%}\n (order by sort_column)\n {%- else -%}\n (partition by {{ group_by | join(\", \") }} order by sort_column)\n {%- endif %} as value_field_lag\n from\n all_values\n\n),\nvalidation_errors as (\n select\n *\n from\n add_lag_values\n where\n value_field_lag is not null\n and\n not (value_field {{ operator }} value_field_lag)\n\n)\nselect *\nfrom validation_errors\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.227082}, "macro.dbt_expectations.test_expect_column_values_to_be_null": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_null", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_null.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_null.sql", "name": "test_expect_column_values_to_be_null", "macro_sql": "{% test expect_column_values_to_be_null(model, column_name, row_condition=None) %}\n\n{% set expression = column_name ~ \" is null\" %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.227641}, "macro.dbt_expectations.test_expect_column_values_to_be_unique": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_unique", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_unique.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_unique.sql", "name": "test_expect_column_values_to_be_unique", "macro_sql": "{% test expect_column_values_to_be_unique(model, column_name, row_condition=None) %}\n{{ dbt_expectations.test_expect_compound_columns_to_be_unique(model, [column_name], row_condition=row_condition) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.test_expect_compound_columns_to_be_unique"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.227992}, "macro.dbt_expectations.test_expect_column_values_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_between.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_between.sql", "name": "test_expect_column_values_to_be_between", "macro_sql": "{% test expect_column_values_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n row_condition=None,\n strictly=False\n ) %}\n\n{% set expression %}\n{{ column_name }}\n{% endset %}\n\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=None,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.22887}, "macro.dbt_expectations.test_expect_column_values_to_be_decreasing": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_decreasing", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_decreasing.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_decreasing.sql", "name": "test_expect_column_values_to_be_decreasing", "macro_sql": "{% test expect_column_values_to_be_decreasing(model, column_name,\n sort_column=None,\n strictly=True,\n row_condition=None,\n group_by=None) %}\n\n{%- set sort_column = column_name if not sort_column else sort_column -%}\n{%- set operator = \"<\" if strictly else \"<=\" %}\nwith all_values as (\n\n select\n {{ sort_column }} as sort_column,\n {%- if group_by -%}\n {{ group_by | join(\", \") }},\n {%- endif %}\n {{ column_name }} as value_field\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nadd_lag_values as (\n\n select\n sort_column,\n value_field,\n lag(value_field) over\n {%- if not group_by -%}\n (order by sort_column)\n {%- else -%}\n (partition by {{ group_by | join(\", \") }} order by sort_column)\n {%- endif %} as value_field_lag\n from\n all_values\n\n),\nvalidation_errors as (\n\n select\n *\n from\n add_lag_values\n where\n value_field_lag is not null\n and\n not (value_field {{ operator }} value_field_lag)\n\n)\nselect *\nfrom validation_errors\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.230221}, "macro.dbt_expectations.test_expect_column_values_to_be_in_type_list": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_in_type_list", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql", "name": "test_expect_column_values_to_be_in_type_list", "macro_sql": "{%- test expect_column_values_to_be_in_type_list(model, column_name, column_type_list) -%}\n{%- if execute -%}\n\n {%- set column_name = column_name | upper -%}\n {%- set columns_in_relation = adapter.get_columns_in_relation(model) -%}\n {%- set column_type_list = column_type_list| map(\"upper\") | list -%}\n with relation_columns as (\n\n {% for column in columns_in_relation %}\n select\n cast('{{ column.name | upper }}' as {{ dbt_utils.type_string() }}) as relation_column,\n cast('{{ column.dtype | upper }}' as {{ dbt_utils.type_string() }}) as relation_column_type\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n ),\n test_data as (\n\n select\n *\n from\n relation_columns\n where\n relation_column = '{{ column_name }}'\n and\n relation_column_type not in ('{{ column_type_list | join(\"', '\") }}')\n\n )\n select *\n from test_data\n\n{%- endif -%}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.type_string"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.231445}, "macro.dbt_expectations.test_expect_column_values_to_be_of_type": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_of_type", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_of_type.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_be_of_type.sql", "name": "test_expect_column_values_to_be_of_type", "macro_sql": "{%- test expect_column_values_to_be_of_type(model, column_name, column_type) -%}\n{{ dbt_expectations.test_expect_column_values_to_be_in_type_list(model, column_name, [column_type]) }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.test_expect_column_values_to_be_in_type_list"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2317681}, "macro.dbt_expectations.test_expect_column_values_to_have_consistent_casing": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_have_consistent_casing", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_have_consistent_casing.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_have_consistent_casing.sql", "name": "test_expect_column_values_to_have_consistent_casing", "macro_sql": "{% test expect_column_values_to_have_consistent_casing(model, column_name, display_inconsistent_columns=False) %}\n\nwith test_data as (\n\n select\n distinct {{ column_name }} as distinct_values\n from\n {{ model }}\n\n ),\n {% if display_inconsistent_columns %}\n validation_errors as (\n\n select\n lower(distinct_values) as inconsistent_columns,\n count(distinct_values) as set_count_case_insensitive\n from\n test_data\n group by 1\n having\n count(distinct_values) > 1\n\n )\n select * from validation_errors\n {% else %}\n validation_errors as (\n\n select\n count(1) as set_count,\n count(distinct lower(distinct_values)) as set_count_case_insensitive\n from\n test_data\n\n )\n select *\n from\n validation_errors\n where\n set_count != set_count_case_insensitive\n {% endif %}\n {%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.232281}, "macro.dbt_expectations.test_expect_column_values_to_not_be_null": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_not_be_null", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/column_values_basic/expect_column_values_to_not_be_null.sql", "original_file_path": "macros/schema_tests/column_values_basic/expect_column_values_to_not_be_null.sql", "name": "test_expect_column_values_to_not_be_null", "macro_sql": "{% test expect_column_values_to_not_be_null(model, column_name, row_condition=None) %}\n\n{% set expression = column_name ~ \" is not null\" %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.232835}, "macro.dbt_expectations.test_expect_column_min_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_min_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_min_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_min_to_be_between.sql", "name": "test_expect_column_min_to_be_between", "macro_sql": "{% test expect_column_min_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\nmin({{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.233803}, "macro.dbt_expectations.test_expect_column_unique_value_count_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_unique_value_count_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_unique_value_count_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_unique_value_count_to_be_between.sql", "name": "test_expect_column_unique_value_count_to_be_between", "macro_sql": "{% test expect_column_unique_value_count_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\ncount(distinct {{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.234727}, "macro.dbt_expectations.test_expect_column_quantile_values_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_quantile_values_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_quantile_values_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_quantile_values_to_be_between.sql", "name": "test_expect_column_quantile_values_to_be_between", "macro_sql": "{% test expect_column_quantile_values_to_be_between(model, column_name,\n quantile,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n\n{% set expression %}\n{{ dbt_expectations.percentile_cont(column_name, quantile) }}\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.percentile_cont", "macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.235719}, "macro.dbt_expectations.test_expect_column_median_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_median_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_median_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_median_to_be_between.sql", "name": "test_expect_column_median_to_be_between", "macro_sql": "{% test expect_column_median_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n\n{% set expression %}\n{{ dbt_expectations.median(column_name) }}\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.median", "macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.236665}, "macro.dbt_expectations.test_expect_column_proportion_of_unique_values_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_proportion_of_unique_values_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_proportion_of_unique_values_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_proportion_of_unique_values_to_be_between.sql", "name": "test_expect_column_proportion_of_unique_values_to_be_between", "macro_sql": "{% test expect_column_proportion_of_unique_values_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\ncount(distinct {{ column_name }})/count({{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2376618}, "macro.dbt_expectations.test_expect_column_distinct_values_to_equal_set": {"unique_id": "macro.dbt_expectations.test_expect_column_distinct_values_to_equal_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql", "name": "test_expect_column_distinct_values_to_equal_set", "macro_sql": "{% test expect_column_distinct_values_to_equal_set(model, column_name,\n value_set,\n quote_values=True,\n row_condition=None\n ) %}\n\nwith all_values as (\n\n select distinct\n {{ column_name }} as column_value\n\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nset_values as (\n\n {% for value in value_set -%}\n select\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif %} as value_field\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n\n),\nunique_set_values as (\n\n select distinct value_field\n from\n set_values\n\n),\nvalidation_errors as (\n\n select\n *\n from\n all_values v\n full outer join\n unique_set_values s on v.column_value = s.value_field\n where\n v.column_value is null or\n s.value_field is null\n\n)\n\nselect *\nfrom validation_errors\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.238682}, "macro.dbt_expectations.test_expect_column_most_common_value_to_be_in_set": {"unique_id": "macro.dbt_expectations.test_expect_column_most_common_value_to_be_in_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql", "name": "test_expect_column_most_common_value_to_be_in_set", "macro_sql": "{% test expect_column_most_common_value_to_be_in_set(model, column_name,\n value_set,\n top_n,\n quote_values=False,\n data_type=\"decimal\",\n row_condition=None\n ) -%}\n {{ adapter.dispatch('test_expect_column_most_common_value_to_be_in_set', 'dbt_expectations') (model, column_name, value_set, top_n, quote_values, data_type, row_condition) }}\n{%- endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_column_most_common_value_to_be_in_set"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.24049}, "macro.dbt_expectations.default__test_expect_column_most_common_value_to_be_in_set": {"unique_id": "macro.dbt_expectations.default__test_expect_column_most_common_value_to_be_in_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_most_common_value_to_be_in_set.sql", "name": "default__test_expect_column_most_common_value_to_be_in_set", "macro_sql": "{% macro default__test_expect_column_most_common_value_to_be_in_set(model, column_name,\n value_set,\n top_n,\n quote_values,\n data_type,\n row_condition\n ) %}\n\nwith value_counts as (\n\n select\n {% if quote_values -%}\n {{ column_name }}\n {%- else -%}\n cast({{ column_name }} as {{ data_type }})\n {%- endif %} as value_field,\n count(*) as value_count\n\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n group by {% if quote_values -%}\n {{ column_name }}\n {%- else -%}\n cast({{ column_name }} as {{ data_type }})\n {%- endif %}\n\n),\nvalue_counts_ranked as (\n\n select\n *,\n row_number() over(order by value_count desc) as value_count_rank\n from\n value_counts\n\n),\nvalue_count_top_n as (\n\n select\n value_field\n from\n value_counts_ranked\n where\n value_count_rank = {{ top_n }}\n\n),\nset_values as (\n\n {% for value in value_set -%}\n select\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n cast({{ value }} as {{ data_type }})\n {%- endif %} as value_field\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n\n),\nunique_set_values as (\n\n select distinct value_field\n from\n set_values\n\n),\nvalidation_errors as (\n -- values from the model that are not in the set\n select\n value_field\n from\n value_count_top_n\n where\n value_field not in (select value_field from unique_set_values)\n\n)\n\nselect *\nfrom validation_errors\n\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.241343}, "macro.dbt_expectations.test_expect_column_distinct_values_to_contain_set": {"unique_id": "macro.dbt_expectations.test_expect_column_distinct_values_to_contain_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql", "name": "test_expect_column_distinct_values_to_contain_set", "macro_sql": "{% test expect_column_distinct_values_to_contain_set(model, column_name,\n value_set,\n quote_values=True,\n row_condition=None\n ) %}\n\nwith all_values as (\n\n select distinct\n {{ column_name }} as value_field\n\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nset_values as (\n\n {% for value in value_set -%}\n select\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif %} as value_field\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n\n),\nunique_set_values as (\n\n select distinct value_field\n from\n set_values\n\n),\nvalidation_errors as (\n -- values in set that are not in the list of values from the model\n select\n s.value_field\n from\n unique_set_values s\n left join\n all_values v on s.value_field = v.value_field\n where\n v.value_field is null\n\n)\n\nselect *\nfrom validation_errors\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.24239}, "macro.dbt_expectations.test_expect_column_distinct_count_to_equal": {"unique_id": "macro.dbt_expectations.test_expect_column_distinct_count_to_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_equal.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_equal.sql", "name": "test_expect_column_distinct_count_to_equal", "macro_sql": "{% test expect_column_distinct_count_to_equal(model,\n column_name,\n value,\n quote_values=False,\n group_by=None,\n row_condition=None\n ) %}\n{% set expression %}\ncount(distinct {{ column_name }}) = {{ value }}\n{% endset %}\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=group_by,\n row_condition=row_condition)\n }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.243136}, "macro.dbt_expectations.test_expect_column_sum_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_sum_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_sum_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_sum_to_be_between.sql", "name": "test_expect_column_sum_to_be_between", "macro_sql": "{% test expect_column_sum_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\nsum({{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.24404}, "macro.dbt_expectations.test_expect_column_stdev_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_stdev_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql", "name": "test_expect_column_stdev_to_be_between", "macro_sql": "{% test expect_column_stdev_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) -%}\n {{ adapter.dispatch('test_expect_column_stdev_to_be_between', 'dbt_expectations') (\n model, column_name,\n min_value,\n max_value,\n group_by,\n row_condition,\n strictly\n ) }}\n{%- endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_column_stdev_to_be_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2455502}, "macro.dbt_expectations.default__test_expect_column_stdev_to_be_between": {"unique_id": "macro.dbt_expectations.default__test_expect_column_stdev_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_stdev_to_be_between.sql", "name": "default__test_expect_column_stdev_to_be_between", "macro_sql": "{% macro default__test_expect_column_stdev_to_be_between(\n model, column_name,\n min_value,\n max_value,\n group_by,\n row_condition,\n strictly\n ) %}\n\n{% set expression %}\nstddev({{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.245972}, "macro.dbt_expectations.test_expect_column_mean_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_mean_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_mean_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_mean_to_be_between.sql", "name": "test_expect_column_mean_to_be_between", "macro_sql": "{% test expect_column_mean_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\navg({{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2468858}, "macro.dbt_expectations.test_expect_column_max_to_be_between": {"unique_id": "macro.dbt_expectations.test_expect_column_max_to_be_between", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_max_to_be_between.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_max_to_be_between.sql", "name": "test_expect_column_max_to_be_between", "macro_sql": "{% test expect_column_max_to_be_between(model, column_name,\n min_value=None,\n max_value=None,\n group_by=None,\n row_condition=None,\n strictly=False\n ) %}\n{% set expression %}\nmax({{ column_name }})\n{% endset %}\n{{ dbt_expectations.expression_between(model,\n expression=expression,\n min_value=min_value,\n max_value=max_value,\n group_by_columns=group_by,\n row_condition=row_condition,\n strictly=strictly\n ) }}\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_between"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.24779}, "macro.dbt_expectations.test_expect_column_distinct_count_to_be_greater_than": {"unique_id": "macro.dbt_expectations.test_expect_column_distinct_count_to_be_greater_than", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_be_greater_than.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_be_greater_than.sql", "name": "test_expect_column_distinct_count_to_be_greater_than", "macro_sql": "{% test expect_column_distinct_count_to_be_greater_than(model,\n column_name,\n value,\n quote_values=False,\n group_by=None,\n row_condition=None\n ) %}\n{% set expression %}\ncount(distinct {{ column_name }}) > {{ value }}\n{% endset %}\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=group_by,\n row_condition=row_condition)\n }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.248565}, "macro.dbt_expectations.test_expect_column_distinct_values_to_be_in_set": {"unique_id": "macro.dbt_expectations.test_expect_column_distinct_values_to_be_in_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql", "name": "test_expect_column_distinct_values_to_be_in_set", "macro_sql": "{% test expect_column_distinct_values_to_be_in_set(model, column_name,\n value_set,\n quote_values=False,\n row_condition=None\n ) %}\n\nwith all_values as (\n\n select distinct\n {{ column_name }} as value_field\n\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n),\nset_values as (\n\n {% for value in value_set -%}\n select\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif %} as value_field\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n\n),\nunique_set_values as (\n\n select distinct value_field\n from\n set_values\n\n),\nvalidation_errors as (\n -- values from the model that are not in the set\n select\n v.value_field\n from\n all_values v\n left join\n unique_set_values s on v.value_field = s.value_field\n where\n s.value_field is null\n\n)\n\nselect *\nfrom validation_errors\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.24961}, "macro.dbt_expectations.test_expect_column_distinct_count_to_equal_other_table": {"unique_id": "macro.dbt_expectations.test_expect_column_distinct_count_to_equal_other_table", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_equal_other_table.sql", "original_file_path": "macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_equal_other_table.sql", "name": "test_expect_column_distinct_count_to_equal_other_table", "macro_sql": "{% test expect_column_distinct_count_to_equal_other_table(model,\n compare_model,\n column_name,\n compare_column_name,\n row_condition=None,\n compare_row_condition=None\n ) %}\n{%- set expression -%}\ncount(distinct {{ column_name }})\n{%- endset -%}\n{%- set compare_expression -%}\n{%- if compare_column_name -%}\ncount(distinct {{ compare_column_name }})\n{%- else -%}\n{{ expression }}\n{%- endif -%}\n{%- endset -%}\n{{ dbt_expectations.test_equal_expression(\n model,\n expression=expression,\n compare_model=compare_model,\n compare_expression=compare_expression,\n row_condition=row_condition,\n compare_row_condition=compare_row_condition\n) }}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.test_equal_expression"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.250515}, "macro.dbt_expectations.test_expect_row_values_to_have_data_for_every_n_datepart": {"unique_id": "macro.dbt_expectations.test_expect_row_values_to_have_data_for_every_n_datepart", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/distributional/expect_row_values_to_have_data_for_every_n_datepart.sql", "original_file_path": "macros/schema_tests/distributional/expect_row_values_to_have_data_for_every_n_datepart.sql", "name": "test_expect_row_values_to_have_data_for_every_n_datepart", "macro_sql": "{%- test expect_row_values_to_have_data_for_every_n_datepart(model,\n date_col,\n date_part=\"day\",\n interval=None,\n row_condition=None,\n exclusion_condition=None,\n test_start_date=None,\n test_end_date=None) -%}\n{% if not execute %}\n {{ return('') }}\n{% endif %}\n\n{% if not test_start_date or not test_end_date %}\n {% set sql %}\n\n select\n min({{ date_col }}) as start_{{ date_part }},\n max({{ date_col }}) as end_{{ date_part }}\n from {{ model }}\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n\n {% endset %}\n\n {%- set dr = run_query(sql) -%}\n {%- set db_start_date = dr.columns[0].values()[0].strftime('%Y-%m-%d') -%}\n {%- set db_end_date = dr.columns[1].values()[0].strftime('%Y-%m-%d') -%}\n\n{% endif %}\n\n{% if not test_start_date %}\n{% set start_date = db_start_date %}\n{% else %}\n{% set start_date = test_start_date %}\n{% endif %}\n\n\n{% if not test_end_date %}\n{% set end_date = db_end_date %}\n{% else %}\n{% set end_date = test_end_date %}\n{% endif %}\nwith base_dates as (\n\n {{ dbt_date.get_base_dates(start_date=start_date, end_date=end_date, datepart=date_part) }}\n {% if interval %}\n {# \n Filter the date spine created above down to the interval granularity using a modulo operation.\n The number of date_parts after the start_date divided by the integer interval will produce no remainder for the desired intervals, \n e.g. for 2-day interval from a starting Jan 1, 2020:\n params: start_date = '2020-01-01', date_part = 'day', interval = 2\n date spine created above: [2020-01-01, 2020-01-02, 2020-01-03, 2020-01-04, 2020-01-05, ...]\n The first parameter to the `mod` function would be the number of days between the start_date and the spine date, i.e. [0, 1, 2, 3, 4 ...]\n The second parameter to the `mod` function would be the integer interval, i.e. 2\n This modulo operation produces the following remainders: [0, 1, 0, 1, 0, ...]\n Filtering the spine only where this remainder == 0 will return a spine with every other day as desired, i.e. [2020-01-01, 2020-01-03, 2020-01-05, ...]\n #}\n where mod(\n cast({{ dbt_utils.datediff(\"'\" ~ start_date ~ \"'\", 'date_' ~ date_part, date_part) }} as {{ dbt_utils.type_int() }}),\n cast({{interval}} as {{ dbt_utils.type_int() }})\n ) = 0\n {% endif %}\n\n),\nmodel_data as (\n\n select\n {% if not interval %}\n\n cast({{ dbt_utils.date_trunc(date_part, date_col) }} as {{ dbt_expectations.type_datetime() }}) as date_{{ date_part }},\n\n {% else %}\n {# \n Use a modulo operator to determine the number of intervals that a date_col is away from the interval-date spine \n and subtracts that amount to effectively slice each date_col record into its corresponding spine bucket,\n e.g. given a date_col of with records [2020-01-01, 2020-01-02, 2020-01-03, 2020-01-11, 2020-01-12]\n if we want to slice these dates into their 2-day buckets starting Jan 1, 2020 (start_date = '2020-01-01', date_part='day', interval=2),\n the modulo operation described above will produce these remainders: [0, 1, 0, 0, 1]\n subtracting that number of days from the observations will produce records [2020-01-01, 2020-01-01, 2020-01-03, 2020-01-11, 2020-01-11],\n all of which align with records from the interval-date spine\n #}\n {{dbt_utils.dateadd(\n date_part, \n \"mod(\n cast(\" ~ dbt_utils.datediff(\"'\" ~ start_date ~ \"'\", date_col, date_part) ~ \" as \" ~ dbt_utils.type_int() ~ \" ),\n cast(\" ~ interval ~ \" as \" ~ dbt_utils.type_int() ~ \" )\n ) * (-1)\", \n \"cast( \" ~ dbt_utils.date_trunc(date_part, date_col) ~ \" as \" ~ dbt_expectations.type_datetime() ~ \")\"\n )}} as date_{{ date_part }},\n \n {% endif %}\n \n count(*) as row_cnt\n from\n {{ model }} f\n {% if row_condition %}\n where {{ row_condition }}\n {% endif %}\n group by\n date_{{date_part}}\n\n),\n\nfinal as (\n\n select\n cast(d.date_{{ date_part }} as {{ dbt_expectations.type_datetime() }}) as date_{{ date_part }},\n case when f.date_{{ date_part }} is null then true else false end as is_missing,\n coalesce(f.row_cnt, 0) as row_cnt\n from\n base_dates d\n left join\n model_data f on cast(d.date_{{ date_part }} as {{ dbt_expectations.type_datetime() }}) = f.date_{{ date_part }}\n)\nselect\n *\nfrom final\nwhere row_cnt = 0\n{% if exclusion_condition %}\n and {{ exclusion_condition }}\n{% endif %}\n{%- endtest -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt_date.get_base_dates", "macro.dbt_utils.datediff", "macro.dbt_utils.type_int", "macro.dbt_utils.date_trunc", "macro.dbt_expectations.type_datetime", "macro.dbt_utils.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.254934}, "macro.dbt_expectations._get_metric_expression": {"unique_id": "macro.dbt_expectations._get_metric_expression", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql", "original_file_path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql", "name": "_get_metric_expression", "macro_sql": "{%- macro _get_metric_expression(metric_column, take_logs) -%}\n\n{%- if take_logs %}\n{%- set expr = \"nullif(\" ~ metric_column ~ \", 0)\" -%}\ncoalesce({{ dbt_expectations.log_natural(expr) }}, 0)\n{%- else -%}\ncoalesce({{ metric_column }}, 0)\n{%- endif %}\n\n{%- endmacro -%}\n\n", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.log_natural"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2586722}, "macro.dbt_expectations.test_expect_column_values_to_be_within_n_moving_stdevs": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_within_n_moving_stdevs", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql", "original_file_path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql", "name": "test_expect_column_values_to_be_within_n_moving_stdevs", "macro_sql": "{% test expect_column_values_to_be_within_n_moving_stdevs(model,\n column_name,\n date_column_name,\n period='day',\n lookback_periods=1,\n trend_periods=7,\n test_periods=14,\n sigma_threshold=3,\n sigma_threshold_upper=None,\n sigma_threshold_lower=None,\n take_diffs=true,\n take_logs=true\n ) -%}\n {{ adapter.dispatch('test_expect_column_values_to_be_within_n_moving_stdevs', 'dbt_expectations') (model,\n column_name,\n date_column_name,\n period,\n lookback_periods,\n trend_periods,\n test_periods,\n sigma_threshold,\n sigma_threshold_upper,\n sigma_threshold_lower,\n take_diffs,\n take_logs\n ) }}\n{%- endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_column_values_to_be_within_n_moving_stdevs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2592561}, "macro.dbt_expectations.default__test_expect_column_values_to_be_within_n_moving_stdevs": {"unique_id": "macro.dbt_expectations.default__test_expect_column_values_to_be_within_n_moving_stdevs", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql", "original_file_path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql", "name": "default__test_expect_column_values_to_be_within_n_moving_stdevs", "macro_sql": "{% macro default__test_expect_column_values_to_be_within_n_moving_stdevs(model,\n column_name,\n date_column_name,\n period,\n lookback_periods,\n trend_periods,\n test_periods,\n sigma_threshold,\n sigma_threshold_upper,\n sigma_threshold_lower,\n take_diffs,\n take_logs\n ) %}\n\n{%- set sigma_threshold_upper = sigma_threshold_upper if sigma_threshold_upper else sigma_threshold -%}\n{%- set sigma_threshold_lower = sigma_threshold_lower if sigma_threshold_lower else -1 * sigma_threshold -%}\n\nwith metric_values as (\n\n with grouped_metric_values as (\n\n select\n {{ dbt_utils.date_trunc(period, date_column_name) }} as metric_period,\n sum({{ column_name }}) as agg_metric_value\n from\n {{ model }}\n group by\n 1\n\n ),\n {%- if take_diffs %}\n grouped_metric_values_with_priors as (\n\n select\n *,\n lag(agg_metric_value, {{ lookback_periods }}) over(order by metric_period) as prior_agg_metric_value\n from\n grouped_metric_values d\n\n )\n select\n *,\n {{ dbt_expectations._get_metric_expression(\"agg_metric_value\", take_logs) }}\n -\n {{ dbt_expectations._get_metric_expression(\"prior_agg_metric_value\", take_logs) }}\n as metric_test_value\n from\n grouped_metric_values_with_priors d\n\n {%- else %}\n\n select\n *,\n {{ dbt_expectations._get_metric_expression(\"agg_metric_value\", take_logs) }}\n from\n grouped_metric_values\n\n {%- endif %}\n\n),\nmetric_moving_calcs as (\n\n select\n *,\n avg(metric_test_value)\n over(order by metric_period rows\n between {{ trend_periods }} preceding and 1 preceding) as metric_test_rolling_average,\n stddev(metric_test_value)\n over(order by metric_period rows\n between {{ trend_periods }} preceding and 1 preceding) as metric_test_rolling_stddev\n from\n metric_values\n\n),\nmetric_sigma as (\n\n select\n *,\n (metric_test_value - metric_test_rolling_average) as metric_test_delta,\n (metric_test_value - metric_test_rolling_average)/nullif(metric_test_rolling_stddev, 0) as metric_test_sigma\n from\n metric_moving_calcs\n\n)\nselect\n *\nfrom\n metric_sigma\nwhere\n\n metric_period >= cast(\n {{ dbt_utils.dateadd(period, -test_periods, dbt_utils.date_trunc(period, dbt_date.now())) }}\n as {{ dbt_utils.type_timestamp() }})\n and\n metric_period < {{ dbt_utils.date_trunc(period, dbt_date.now()) }}\n and\n\n not (\n metric_test_sigma >= {{ sigma_threshold_lower }} and\n metric_test_sigma <= {{ sigma_threshold_upper }}\n )\n{%- endmacro -%}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_utils.date_trunc", "macro.dbt_expectations._get_metric_expression", "macro.dbt_utils.dateadd", "macro.dbt_date.now", "macro.dbt_utils.type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2604978}, "macro.dbt_expectations.test_expect_column_values_to_be_within_n_stdevs": {"unique_id": "macro.dbt_expectations.test_expect_column_values_to_be_within_n_stdevs", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql", "original_file_path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql", "name": "test_expect_column_values_to_be_within_n_stdevs", "macro_sql": "{% test expect_column_values_to_be_within_n_stdevs(model,\n column_name,\n group_by=None,\n sigma_threshold=3\n ) -%}\n {{ adapter.dispatch('test_expect_column_values_to_be_within_n_stdevs', 'dbt_expectations') (model, column_name, group_by, sigma_threshold) }}\n{%- endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_column_values_to_be_within_n_stdevs"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.261562}, "macro.dbt_expectations.default__test_expect_column_values_to_be_within_n_stdevs": {"unique_id": "macro.dbt_expectations.default__test_expect_column_values_to_be_within_n_stdevs", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql", "original_file_path": "macros/schema_tests/distributional/expect_column_values_to_be_within_n_stdevs.sql", "name": "default__test_expect_column_values_to_be_within_n_stdevs", "macro_sql": "{% macro default__test_expect_column_values_to_be_within_n_stdevs(model,\n column_name,\n group_by,\n sigma_threshold\n ) %}\nwith metric_values as (\n\n {% if group_by -%}\n select\n {{ group_by }} as metric_date,\n sum({{ column_name }}) as {{ column_name }}\n from\n {{ model }}\n group by\n 1\n {%- else -%}\n select\n {{ column_name }} as {{ column_name }}\n from\n {{ model }}\n {%- endif %}\n\n),\nmetric_values_with_statistics as (\n\n select\n *,\n avg({{ column_name }}) over() as {{ column_name }}_average,\n stddev({{ column_name }}) over() as {{ column_name }}_stddev\n from\n metric_values\n\n),\nmetric_values_z_scores as (\n\n select\n *,\n ({{ column_name }} - {{ column_name }}_average)/{{ column_name }}_stddev as {{ column_name }}_sigma\n from\n metric_values_with_statistics\n\n)\nselect\n *\nfrom\n metric_values_z_scores\nwhere\n abs({{ column_name }}_sigma) > {{ sigma_threshold }}\n{%- endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2621472}, "macro.dbt_expectations.test_expect_select_column_values_to_be_unique_within_record": {"unique_id": "macro.dbt_expectations.test_expect_select_column_values_to_be_unique_within_record", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_select_column_values_to_be_unique_within_record.sql", "original_file_path": "macros/schema_tests/multi-column/expect_select_column_values_to_be_unique_within_record.sql", "name": "test_expect_select_column_values_to_be_unique_within_record", "macro_sql": "{% test expect_select_column_values_to_be_unique_within_record(model,\n column_list,\n quote_columns=False,\n ignore_row_if=\"all_values_are_missing\",\n row_condition=None\n ) -%}\n {{ adapter.dispatch('test_expect_select_column_values_to_be_unique_within_record', 'dbt_expectations') (model, column_list, quote_columns, ignore_row_if, row_condition) }}\n{%- endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.default__test_expect_select_column_values_to_be_unique_within_record"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2641568}, "macro.dbt_expectations.default__test_expect_select_column_values_to_be_unique_within_record": {"unique_id": "macro.dbt_expectations.default__test_expect_select_column_values_to_be_unique_within_record", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_select_column_values_to_be_unique_within_record.sql", "original_file_path": "macros/schema_tests/multi-column/expect_select_column_values_to_be_unique_within_record.sql", "name": "default__test_expect_select_column_values_to_be_unique_within_record", "macro_sql": "{% macro default__test_expect_select_column_values_to_be_unique_within_record(model,\n column_list,\n quote_columns,\n ignore_row_if,\n row_condition\n ) %}\n\n{% if not quote_columns %}\n {%- set columns=column_list %}\n{% elif quote_columns %}\n {%- set columns=[] %}\n {% for column in column_list -%}\n {% set columns = columns.append( adapter.quote(column) ) %}\n {%- endfor %}\n{% else %}\n {{ exceptions.raise_compiler_error(\n \"`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '\" ~ quote_columns ~\"'.'\"\n ) }}\n{% endif %}\n\nwith column_values as (\n\n select\n row_number() over(order by 1) as row_index,\n {% for column in columns -%}\n {{ column }}{% if not loop.last %},{% endif %}\n {%- endfor %}\n from {{ model }}\n where 1=1\n {% if row_condition %}\n and {{ row_condition }}\n {% endif %}\n {% if ignore_row_if == \"all_values_are_missing\" %}\n and\n (\n {% for column in columns -%}\n {{ column }} is not null{% if not loop.last %} and {% endif %}\n {%- endfor %}\n )\n {% elif ignore_row_if == \"any_value_is_missing\" %}\n and\n (\n {% for column in columns -%}\n {{ column }} is not null{% if not loop.last %} or {% endif %}\n {%- endfor %}\n )\n {% endif %}\n\n),\nunpivot_columns as (\n\n {% for column in columns %}\n select row_index, '{{ column }}' as column_name, {{ column }} as column_value from column_values\n {% if not loop.last %}union all{% endif %}\n {% endfor %}\n),\nvalidation_errors as (\n\n select\n row_index,\n count(distinct column_value) as column_values\n from unpivot_columns\n group by 1\n having count(distinct column_value) < {{ columns | length }}\n\n)\nselect * from validation_errors\n{% endmacro %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2656069}, "macro.dbt_expectations.test_expect_compound_columns_to_be_unique": {"unique_id": "macro.dbt_expectations.test_expect_compound_columns_to_be_unique", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_compound_columns_to_be_unique.sql", "original_file_path": "macros/schema_tests/multi-column/expect_compound_columns_to_be_unique.sql", "name": "test_expect_compound_columns_to_be_unique", "macro_sql": "{% test expect_compound_columns_to_be_unique(model,\n column_list,\n quote_columns=False,\n ignore_row_if=\"all_values_are_missing\",\n row_condition=None\n ) %}\n{% if not column_list %}\n {{ exceptions.raise_compiler_error(\n \"`column_list` must be specified as a list of columns. Got: '\" ~ column_list ~\"'.'\"\n ) }}\n{% endif %}\n{% if not quote_columns %}\n {%- set columns=column_list %}\n{% elif quote_columns %}\n {%- set columns=[] %}\n {% for column in column_list -%}\n {% set columns = columns.append( adapter.quote(column) ) %}\n {%- endfor %}\n{% else %}\n {{ exceptions.raise_compiler_error(\n \"`quote_columns` argument for expect_compound_columns_to_be_unique test must be one of [True, False] Got: '\" ~ quote_columns ~\"'.'\"\n ) }}\n{% endif %}\n\n{%- set row_condition_ext -%}\n\n{%- if row_condition %}\n {{ row_condition }} and\n{% endif -%}\n\n{%- if ignore_row_if == \"all_values_are_missing\" %}\n (\n {% for column in columns -%}\n {{ column }} is not null{% if not loop.last %} and {% endif %}\n {% endfor %}\n )\n{%- elif ignore_row_if == \"any_value_is_missing\" %}\n (\n {% for column in columns -%}\n {{ column }} is not null{% if not loop.last %} or {% endif %}\n {% endfor %}\n )\n{%- endif -%}\n{%- endset -%}\n\nwith validation_errors as (\n\n select\n {% for column in columns -%}\n {{ column }}{% if not loop.last %},{% endif %}\n {%- endfor %}\n from {{ model }}\n where\n 1=1\n {%- if row_condition_ext %}\n and {{ row_condition_ext }}\n {% endif %}\n group by\n {% for column in columns -%}\n {{ column }}{% if not loop.last %},{% endif %}\n {%- endfor %}\n having count(*) > 1\n\n)\nselect * from validation_errors\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.268668}, "macro.dbt_expectations.test_expect_multicolumn_sum_to_equal": {"unique_id": "macro.dbt_expectations.test_expect_multicolumn_sum_to_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_multicolumn_sum_to_equal.sql", "original_file_path": "macros/schema_tests/multi-column/expect_multicolumn_sum_to_equal.sql", "name": "test_expect_multicolumn_sum_to_equal", "macro_sql": "{% test expect_multicolumn_sum_to_equal(model,\n column_list,\n sum_total,\n group_by=None,\n row_condition=None\n ) %}\n\n{% set expression %}\n{% for column in column_list %}\nsum({{ column }}){% if not loop.last %} + {% endif %}\n{% endfor %} = {{ sum_total }}\n{% endset %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=group_by,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.269741}, "macro.dbt_expectations.test_expect_column_pair_values_to_be_equal": {"unique_id": "macro.dbt_expectations.test_expect_column_pair_values_to_be_equal", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_column_pair_values_to_be_equal.sql", "original_file_path": "macros/schema_tests/multi-column/expect_column_pair_values_to_be_equal.sql", "name": "test_expect_column_pair_values_to_be_equal", "macro_sql": "{% test expect_column_pair_values_to_be_equal(model,\n column_A,\n column_B,\n row_condition=None\n ) %}\n\n{% set operator = \"=\" %}\n{% set expression = column_A ~ \" \" ~ operator ~ \" \" ~ column_B %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.2875159}, "macro.dbt_expectations.test_expect_column_pair_values_A_to_be_greater_than_B": {"unique_id": "macro.dbt_expectations.test_expect_column_pair_values_A_to_be_greater_than_B", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_column_pair_values_A_to_be_greater_than_B.sql", "original_file_path": "macros/schema_tests/multi-column/expect_column_pair_values_A_to_be_greater_than_B.sql", "name": "test_expect_column_pair_values_A_to_be_greater_than_B", "macro_sql": "{% test expect_column_pair_values_A_to_be_greater_than_B(model,\n column_A,\n column_B,\n or_equal=False,\n row_condition=None\n ) %}\n\n{% set operator = \">=\" if or_equal else \">\" %}\n{% set expression = column_A ~ \" \" ~ operator ~ \" \" ~ column_B %}\n\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.288492}, "macro.dbt_expectations.test_expect_column_pair_values_to_be_in_set": {"unique_id": "macro.dbt_expectations.test_expect_column_pair_values_to_be_in_set", "package_name": "dbt_expectations", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop/dbt_packages/dbt_expectations", "path": "macros/schema_tests/multi-column/expect_column_pair_values_to_be_in_set.sql", "original_file_path": "macros/schema_tests/multi-column/expect_column_pair_values_to_be_in_set.sql", "name": "test_expect_column_pair_values_to_be_in_set", "macro_sql": "{% test expect_column_pair_values_to_be_in_set(model,\n column_A,\n column_B,\n value_pairs_set,\n row_condition=None\n ) %}\n\n{% set expression %}\n{% for pair in value_pairs_set %}\n{%- if (pair | length) == 2 %}\n({{ column_A }} = {{ pair[0] }} and {{ column_B }} = {{ pair[1] }}){% if not loop.last %} or {% endif %}\n{% else %}\n{{ exceptions.raise_compiler_error(\n \"`value_pairs_set` argument for expect_column_pair_values_to_be_in_set test cannot have more than 2 item per element.\n Got: '\" ~ pair ~ \"'.'\"\n ) }}\n{% endif %}\n{% endfor %}\n{% endset %}\n{{ dbt_expectations.expression_is_true(model,\n expression=expression,\n group_by_columns=None,\n row_condition=row_condition\n )\n }}\n\n{% endtest %}", "resource_type": "macro", "tags": [], "depends_on": {"macros": ["macro.dbt_expectations.expression_is_true"]}, "description": "", "meta": {}, "docs": {"show": true}, "patch_path": null, "arguments": [], "created_at": 1654986671.290306}}, "docs": {"jaffle_shop.__overview__": {"unique_id": "jaffle_shop.__overview__", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "overview.md", "original_file_path": "models/overview.md", "name": "__overview__", "block_contents": "## Data Documentation for Jaffle Shop\n\n`jaffle_shop` is a fictional ecommerce store.\n\nThis [dbt](https://www.getdbt.com/) project is for testing out code.\n\nThe source code can be found [here](https://github.com/clrcrl/jaffle_shop)."}, "jaffle_shop.orders_status": {"unique_id": "jaffle_shop.orders_status", "package_name": "jaffle_shop", "root_path": "/Users/shirshankadas/workspace/dbt_work/jaffle_shop", "path": "docs.md", "original_file_path": "models/docs.md", "name": "orders_status", "block_contents": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |"}, "dbt.__overview__": {"unique_id": "dbt.__overview__", "package_name": "dbt", "root_path": "/Users/shirshankadas/workspace/dbt_work/venv/lib/python3.9/site-packages/dbt/include/global_project", "path": "overview.md", "original_file_path": "docs/overview.md", "name": "__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.jaffle_shop.stg_customers": ["seed.jaffle_shop.raw_customers"], "model.jaffle_shop.stg_payments": ["seed.jaffle_shop.raw_payments"], "model.jaffle_shop.stg_orders": ["seed.jaffle_shop.raw_orders"], "test.jaffle_shop.assert_total_payment_amount_is_positive": ["model.jaffle_shop.orders"], "seed.jaffle_shop.raw_customers": [], "seed.jaffle_shop.raw_orders": [], "seed.jaffle_shop.raw_payments": [], "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": ["model.jaffle_shop.stg_customers"], "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": ["model.jaffle_shop.stg_customers"], "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": ["model.jaffle_shop.stg_orders"], "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": ["model.jaffle_shop.stg_orders"], "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": ["model.jaffle_shop.stg_orders"], "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": ["model.jaffle_shop.stg_payments"], "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": ["model.jaffle_shop.stg_payments"], "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": ["model.jaffle_shop.stg_payments"], "model.jaffle_shop.customers": ["model.jaffle_shop.stg_customers", "model.jaffle_shop.stg_orders", "model.jaffle_shop.stg_payments"], "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": ["model.jaffle_shop.customers"], "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": ["model.jaffle_shop.customers"], "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False.e67667298f": ["model.jaffle_shop.customers"], "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2.81450cfcd8": ["model.jaffle_shop.customers"], "model.jaffle_shop.orders": ["model.jaffle_shop.stg_orders", "model.jaffle_shop.stg_payments"], "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": ["model.jaffle_shop.orders"], "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": ["model.jaffle_shop.customers", "model.jaffle_shop.orders"], "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_amount.106140f9fd": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": ["model.jaffle_shop.orders"], "test.jaffle_shop.dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0.888b06036c": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": ["model.jaffle_shop.orders"], "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": ["model.jaffle_shop.orders"]}, "child_map": {"model.jaffle_shop.stg_customers": ["model.jaffle_shop.customers", "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada"], "model.jaffle_shop.stg_payments": ["model.jaffle_shop.customers", "model.jaffle_shop.orders", "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", "test.jaffle_shop.unique_stg_payments_payment_id.3744510712"], "model.jaffle_shop.stg_orders": ["model.jaffle_shop.customers", "model.jaffle_shop.orders", "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a"], "test.jaffle_shop.assert_total_payment_amount_is_positive": [], "seed.jaffle_shop.raw_customers": ["model.jaffle_shop.stg_customers"], "seed.jaffle_shop.raw_orders": ["model.jaffle_shop.stg_orders"], "seed.jaffle_shop.raw_payments": ["model.jaffle_shop.stg_payments"], "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": [], "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": [], "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": [], "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": [], "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": [], "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": [], "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [], "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": [], "model.jaffle_shop.customers": ["test.jaffle_shop.dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False.e67667298f", "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2.81450cfcd8", "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1"], "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [], "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": [], "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False.e67667298f": [], "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2.81450cfcd8": [], "model.jaffle_shop.orders": ["test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", "test.jaffle_shop.assert_total_payment_amount_is_positive", "test.jaffle_shop.dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0.888b06036c", "test.jaffle_shop.not_null_orders_amount.106140f9fd", "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", "test.jaffle_shop.unique_orders_order_id.fed79b3a6e"], "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": [], "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": [], "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [], "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [], "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": [], "test.jaffle_shop.not_null_orders_amount.106140f9fd": [], "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [], "test.jaffle_shop.dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0.888b06036c": [], "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": [], "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": [], "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": []}} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/jaffle_shop_test_results.json b/metadata-ingestion/tests/integration/dbt/jaffle_shop_test_results.json new file mode 100644 index 00000000000000..491775d7b5caf2 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/jaffle_shop_test_results.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v4.json", "dbt_version": "1.1.0", "generated_at": "2022-06-18T15:12:17.668663Z", "invocation_id": "c7a6b778-0e0f-4789-b567-ca7e124a6840", "env": {}}, "results": [{"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:11.035800Z", "completed_at": "2022-06-18T15:12:11.058025Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:11.075728Z", "completed_at": "2022-06-18T15:12:12.430351Z"}], "thread_id": "Thread-2", "execution_time": 1.3975439071655273, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:11.035961Z", "completed_at": "2022-06-18T15:12:11.058146Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:11.077306Z", "completed_at": "2022-06-18T15:12:12.431694Z"}], "thread_id": "Thread-4", "execution_time": 1.3975999355316162, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.assert_total_payment_amount_is_positive"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:11.035875Z", "completed_at": "2022-06-18T15:12:11.057870Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:11.073626Z", "completed_at": "2022-06-18T15:12:12.453347Z"}], "thread_id": "Thread-3", "execution_time": 1.4187071323394775, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:11.035712Z", "completed_at": "2022-06-18T15:12:11.057667Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:11.058878Z", "completed_at": "2022-06-18T15:12:12.553348Z"}], "thread_id": "Thread-1", "execution_time": 1.5191490650177002, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3"}, {"status": "fail", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:12.435775Z", "completed_at": "2022-06-18T15:12:12.454362Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:12.458287Z", "completed_at": "2022-06-18T15:12:13.567413Z"}], "thread_id": "Thread-2", "execution_time": 1.134261131286621, "adapter_response": {}, "message": "Got 80 results, configured to fail if != 0", "failures": 80, "unique_id": "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_between_customers_customer_id__20__0__customer_id_is_not_null__False.5702665c39"}, {"status": "error", "timing": [], "thread_id": "Thread-4", "execution_time": 1.1353793144226074, "adapter_response": {}, "message": "Database Error in test dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2 (models/schema.yml)\n No matching signature for operator = for argument types: INT64, STRING. Supported signature: ANY = ANY at [46:25]\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql", "failures": null, "unique_id": "test.jaffle_shop.dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2.81450cfcd8"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:12.554764Z", "completed_at": "2022-06-18T15:12:12.560185Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:12.560375Z", "completed_at": "2022-06-18T15:12:13.574237Z"}], "thread_id": "Thread-1", "execution_time": 1.0206811428070068, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d"}, {"status": "error", "timing": [], "thread_id": "Thread-3", "execution_time": 1.5632309913635254, "adapter_response": {}, "message": "Database Error in test dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0 (models/schema.yml)\n No matching signature for operator = for argument types: FLOAT64, STRING. Supported signature: ANY = ANY at [36:25]\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql", "failures": null, "unique_id": "test.jaffle_shop.dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0.888b06036c"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:13.575762Z", "completed_at": "2022-06-18T15:12:13.585229Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:13.585898Z", "completed_at": "2022-06-18T15:12:14.462716Z"}], "thread_id": "Thread-2", "execution_time": 0.8910889625549316, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_amount.106140f9fd"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:13.585707Z", "completed_at": "2022-06-18T15:12:13.592438Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:13.595233Z", "completed_at": "2022-06-18T15:12:14.464168Z"}], "thread_id": "Thread-1", "execution_time": 0.8804700374603271, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:13.581430Z", "completed_at": "2022-06-18T15:12:13.585791Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:13.591247Z", "completed_at": "2022-06-18T15:12:14.474123Z"}], "thread_id": "Thread-4", "execution_time": 0.9056179523468018, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:14.025274Z", "completed_at": "2022-06-18T15:12:14.031457Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:14.031853Z", "completed_at": "2022-06-18T15:12:14.957755Z"}], "thread_id": "Thread-3", "execution_time": 0.9345941543579102, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:14.470280Z", "completed_at": "2022-06-18T15:12:14.482079Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:14.482587Z", "completed_at": "2022-06-18T15:12:15.354110Z"}], "thread_id": "Thread-2", "execution_time": 0.8874030113220215, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_customer_id.c5f02694af"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:14.470571Z", "completed_at": "2022-06-18T15:12:14.482156Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:14.485355Z", "completed_at": "2022-06-18T15:12:15.365747Z"}], "thread_id": "Thread-1", "execution_time": 0.8978712558746338, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:14.482489Z", "completed_at": "2022-06-18T15:12:14.492294Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:14.493905Z", "completed_at": "2022-06-18T15:12:15.498329Z"}], "thread_id": "Thread-4", "execution_time": 1.0170459747314453, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_orders_order_id.cf6c17daed"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:14.961696Z", "completed_at": "2022-06-18T15:12:14.966381Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:14.966557Z", "completed_at": "2022-06-18T15:12:15.821731Z"}], "thread_id": "Thread-3", "execution_time": 0.8619332313537598, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:15.357737Z", "completed_at": "2022-06-18T15:12:15.367350Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:15.368302Z", "completed_at": "2022-06-18T15:12:16.209056Z"}], "thread_id": "Thread-2", "execution_time": 0.8549778461456299, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:15.371334Z", "completed_at": "2022-06-18T15:12:15.377469Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:15.377864Z", "completed_at": "2022-06-18T15:12:16.211199Z"}], "thread_id": "Thread-1", "execution_time": 0.8449010848999023, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:15.500987Z", "completed_at": "2022-06-18T15:12:15.510017Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:15.510218Z", "completed_at": "2022-06-18T15:12:16.389583Z"}], "thread_id": "Thread-4", "execution_time": 0.8895509243011475, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:15.825588Z", "completed_at": "2022-06-18T15:12:15.836145Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:15.836502Z", "completed_at": "2022-06-18T15:12:16.690091Z"}], "thread_id": "Thread-3", "execution_time": 0.8856301307678223, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:16.217031Z", "completed_at": "2022-06-18T15:12:16.229927Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:16.269200Z", "completed_at": "2022-06-18T15:12:17.195033Z"}], "thread_id": "Thread-2", "execution_time": 0.9806849956512451, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.unique_orders_order_id.fed79b3a6e"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:16.217232Z", "completed_at": "2022-06-18T15:12:16.229780Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:16.230402Z", "completed_at": "2022-06-18T15:12:17.319119Z"}], "thread_id": "Thread-1", "execution_time": 1.1094379425048828, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:16.391329Z", "completed_at": "2022-06-18T15:12:16.395508Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:16.395771Z", "completed_at": "2022-06-18T15:12:17.341560Z"}], "thread_id": "Thread-4", "execution_time": 0.9649977684020996, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a"}, {"status": "pass", "timing": [{"name": "compile", "started_at": "2022-06-18T15:12:16.714216Z", "completed_at": "2022-06-18T15:12:16.719360Z"}, {"name": "execute", "started_at": "2022-06-18T15:12:16.719670Z", "completed_at": "2022-06-18T15:12:17.664336Z"}], "thread_id": "Thread-3", "execution_time": 0.9523539543151855, "adapter_response": {}, "message": null, "failures": 0, "unique_id": "test.jaffle_shop.unique_stg_payments_payment_id.3744510712"}], "elapsed_time": 7.126053094863892, "args": {"write_json": true, "use_colors": true, "printer_width": 80, "version_check": true, "partial_parse": true, "static_parser": true, "profiles_dir": "/Users/shirshankadas/.dbt", "send_anonymous_usage_stats": true, "event_buffer_size": 100000, "quiet": false, "no_print": false, "indirect_selection": "eager", "which": "test", "rpc_method": "test"}} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index ebebe0a3da17ad..cf48c607c55bfc 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -1,14 +1,26 @@ from os import PathLike -from typing import Any, Dict, Optional, Union, cast +from typing import Any, Dict, Optional, Type, Union, cast from unittest.mock import patch import pytest import requests_mock from freezegun import freeze_time -from datahub.ingestion.run.pipeline import Pipeline -from datahub.ingestion.source.dbt import DBTSource -from datahub.ingestion.source.state.checkpoint import Checkpoint +from datahub.configuration.common import DynamicTypedConfig +from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import JobId +from datahub.ingestion.run.pipeline import Pipeline, PipelineConfig, SourceConfig +from datahub.ingestion.source.dbt import ( + DBTConfig, + DBTEntitiesEnabled, + DBTSource, + EmitDirective, +) +from datahub.ingestion.source.sql.sql_types import ( + TRINO_SQL_TYPES_MAP, + resolve_trino_modified_type, +) +from datahub.ingestion.source.state.checkpoint import Checkpoint, CheckpointStateBase +from datahub.ingestion.source.state.dbt_state import DbtCheckpointState from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, ) @@ -61,6 +73,11 @@ def __init__( "enable_meta_mapping": False, "write_semantics": "OVERRIDE", "meta_mapping": { + "owner": { + "match": "^@(.*)", + "operation": "add_owner", + "config": {"owner_type": "user"}, + }, "business_owner": { "match": ".*", "operation": "add_owner", @@ -253,6 +270,18 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): "platform_instance": "dbt-instance-1", }, ), + DbtTestConfig( + "dbt-test-with-target-platform-instance", + test_resources_dir, + test_resources_dir, + tmp_path, + "dbt_test_with_target_platform_instance_mces.json", + "dbt_test_with_target_platform_instance_mces_golden.json", + source_config_modifiers={ + "load_schemas": True, + "target_platform_instance": "ps-instance-1", + }, + ), ] for config in config_variants: @@ -383,9 +412,9 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): # Perform all assertions on the states. The deleted table should not be # part of the second state - state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state) - state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state) - difference_urns = list(state1.get_table_urns_not_in(state2)) + state1 = cast(DbtCheckpointState, checkpoint1.state) + state2 = cast(DbtCheckpointState, checkpoint2.state) + difference_urns = list(state1.get_node_urns_not_in(state2)) assert len(difference_urns) == 2 @@ -409,3 +438,395 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): output_path=deleted_mces_path, golden_path=deleted_actor_golden_mcs, ) + + +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_state_backward_compatibility( + pytestconfig, tmp_path, mock_time, mock_datahub_graph +): + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + manifest_path = f"{test_resources_dir}/dbt_manifest.json" + catalog_path = f"{test_resources_dir}/dbt_catalog.json" + sources_path = f"{test_resources_dir}/dbt_sources.json" + + stateful_config: Dict[str, Any] = { + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + } + + scd_config: Dict[str, Any] = { + "manifest_path": manifest_path, + "catalog_path": catalog_path, + "sources_path": sources_path, + "target_platform": "postgres", + "load_schemas": True, + # This will bypass check in get_workunits function of dbt.py + "write_semantics": "OVERRIDE", + "owner_extraction_pattern": r"^@(?P(.*))", + # enable stateful ingestion + **stateful_config, + } + + pipeline_config_dict: Dict[str, Any] = { + "source": { + "type": "dbt", + "config": scd_config, + }, + "sink": { + # we are not really interested in the resulting events for this test + "type": "console" + }, + "pipeline_name": "statefulpipeline", + } + + with patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + pipeline = Pipeline.create(pipeline_config_dict) + dbt_source = cast(DBTSource, pipeline.source) + + def get_fake_base_sql_alchemy_checkpoint_state( + job_id: JobId, checkpoint_state_class: Type[CheckpointStateBase] + ) -> Optional[Checkpoint]: + if checkpoint_state_class is DbtCheckpointState: + raise Exception( + "DBT source will call this function again with BaseSQLAlchemyCheckpointState" + ) + + sql_state = BaseSQLAlchemyCheckpointState() + urn1 = "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)" + urn2 = ( + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)" + ) + sql_state.add_table_urn(urn1) + sql_state.add_table_urn(urn2) + + assert dbt_source.ctx.pipeline_name is not None + + return Checkpoint( + job_name=dbt_source.get_default_ingestion_job_id(), + pipeline_name=dbt_source.ctx.pipeline_name, + platform_instance_id=dbt_source.get_platform_instance_id(), + run_id=dbt_source.ctx.run_id, + config=dbt_source.config, + state=sql_state, + ) + + # Set fake method to return BaseSQLAlchemyCheckpointState + dbt_source.get_last_checkpoint = get_fake_base_sql_alchemy_checkpoint_state # type: ignore[assignment] + last_checkpoint = dbt_source.get_last_dbt_checkpoint( + dbt_source.get_default_ingestion_job_id(), DbtCheckpointState + ) + # Our fake method is returning BaseSQLAlchemyCheckpointState,however it should get converted to DbtCheckpointState + assert last_checkpoint is not None and isinstance( + last_checkpoint.state, DbtCheckpointState + ) + + pipeline.run() + pipeline.raise_from_status() + + +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_tests(pytestconfig, tmp_path, mock_time, **kwargs): + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + + # Run the metadata ingestion pipeline. + output_file = tmp_path / "dbt_test_events.json" + golden_path = test_resources_dir / "dbt_test_events_golden.json" + + pipeline = Pipeline( + config=PipelineConfig( + source=SourceConfig( + type="dbt", + config=DBTConfig( + manifest_path=str( + (test_resources_dir / "jaffle_shop_manifest.json").resolve() + ), + catalog_path=str( + (test_resources_dir / "jaffle_shop_catalog.json").resolve() + ), + target_platform="postgres", + delete_tests_as_datasets=True, + test_results_path=str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ), + # this is just here to avoid needing to access datahub server + write_semantics="OVERRIDE", + ), + ), + sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}), + ) + ) + pipeline.run() + pipeline.raise_from_status() + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_file, + golden_path=golden_path, + ignore_paths=[], + ) + + +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_stateful_tests(pytestconfig, tmp_path, mock_time, mock_datahub_graph): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + output_file = tmp_path / "dbt_stateful_tests.json" + golden_path = test_resources_dir / "dbt_stateful_tests_golden.json" + manifest_path = str((test_resources_dir / "jaffle_shop_manifest.json").resolve()) + catalog_path = str((test_resources_dir / "jaffle_shop_catalog.json").resolve()) + test_results_path = str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ) + + stateful_config = { + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + } + + scd: Dict[str, Any] = { + "manifest_path": manifest_path, + "catalog_path": catalog_path, + "test_results_path": test_results_path, + "target_platform": "postgres", + "load_schemas": True, + # This will bypass check in get_workunits function of dbt.py + "write_semantics": "OVERRIDE", + "owner_extraction_pattern": r"^@(?P(.*))", + # enable stateful ingestion + **stateful_config, + } + + pipeline_config_dict: Dict[str, Any] = { + "source": { + "type": "dbt", + "config": scd, + }, + "sink": { + # we are not really interested in the resulting events for this test + "type": "file", + "config": {"filename": str(output_file)}, + }, + "pipeline_name": "statefulpipeline", + "run_id": "test_pipeline", + } + + with patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + pipeline = Pipeline.create(pipeline_config_dict) + pipeline.run() + pipeline.raise_from_status() + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_file, + golden_path=golden_path, + ignore_paths=[], + ) + + +@pytest.mark.parametrize( + "data_type, expected_data_type", + [ + ("boolean", "boolean"), + ("tinyint", "tinyint"), + ("smallint", "smallint"), + ("int", "int"), + ("integer", "integer"), + ("bigint", "bigint"), + ("real", "real"), + ("double", "double"), + ("decimal(10,0)", "decimal"), + ("varchar(20)", "varchar"), + ("char", "char"), + ("varbinary", "varbinary"), + ("json", "json"), + ("date", "date"), + ("time", "time"), + ("time(12)", "time"), + ("timestamp", "timestamp"), + ("timestamp(3)", "timestamp"), + ("row(x bigint, y double)", "row"), + ], +) +def test_resolve_trino_modified_type(data_type, expected_data_type): + assert ( + resolve_trino_modified_type(data_type) + == TRINO_SQL_TYPES_MAP[expected_data_type] + ) + + +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_tests_only_assertions(pytestconfig, tmp_path, mock_time, **kwargs): + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + + # Run the metadata ingestion pipeline. + output_file = tmp_path / "test_only_assertions.json" + + pipeline = Pipeline( + config=PipelineConfig( + source=SourceConfig( + type="dbt", + config=DBTConfig( + manifest_path=str( + (test_resources_dir / "jaffle_shop_manifest.json").resolve() + ), + catalog_path=str( + (test_resources_dir / "jaffle_shop_catalog.json").resolve() + ), + target_platform="postgres", + delete_tests_as_datasets=True, + test_results_path=str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ), + # this is just here to avoid needing to access datahub server + write_semantics="OVERRIDE", + entities_enabled=DBTEntitiesEnabled( + test_results=EmitDirective.ONLY + ), + ), + ), + sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}), + ) + ) + pipeline.run() + pipeline.raise_from_status() + # Verify the output. + # No datasets were emitted, and more than 20 events were emitted + assert ( + mce_helpers.assert_entity_urn_not_like( + entity_type="dataset", + regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt", + file=output_file, + ) + > 20 + ) + number_of_valid_assertions_in_test_results = 23 + assert ( + mce_helpers.assert_entity_urn_like( + entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file + ) + == number_of_valid_assertions_in_test_results + ) + # no assertionInfo should be emitted + try: + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionInfo", + aspect_field_matcher={}, + file=output_file, + ) + except AssertionError: + pass + + # all assertions must have an assertionRunEvent emitted (except for one assertion) + assert ( + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionRunEvent", + aspect_field_matcher={}, + file=output_file, + exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"], + ) + == number_of_valid_assertions_in_test_results + ) + + +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_only_test_definitions_and_results( + pytestconfig, tmp_path, mock_time, **kwargs +): + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + + # Run the metadata ingestion pipeline. + output_file = tmp_path / "test_only_definitions_and_assertions.json" + + pipeline = Pipeline( + config=PipelineConfig( + source=SourceConfig( + type="dbt", + config=DBTConfig( + manifest_path=str( + (test_resources_dir / "jaffle_shop_manifest.json").resolve() + ), + catalog_path=str( + (test_resources_dir / "jaffle_shop_catalog.json").resolve() + ), + target_platform="postgres", + test_results_path=str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ), + # this is just here to avoid needing to access datahub server + write_semantics="OVERRIDE", + entities_enabled=DBTEntitiesEnabled( + sources=EmitDirective.NO, + seeds=EmitDirective.NO, + models=EmitDirective.NO, + ), + ), + ), + sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}), + ) + ) + pipeline.run() + pipeline.raise_from_status() + # Verify the output. No datasets were emitted + assert ( + mce_helpers.assert_entity_urn_not_like( + entity_type="dataset", + regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt", + file=output_file, + ) + > 20 + ) + number_of_assertions = 24 + assert ( + mce_helpers.assert_entity_urn_like( + entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file + ) + == number_of_assertions + ) + # all assertions must have an assertionInfo emitted + assert ( + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionInfo", + aspect_field_matcher={}, + file=output_file, + ) + == number_of_assertions + ) + # all assertions must have an assertionRunEvent emitted (except for one assertion) + assert ( + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionRunEvent", + aspect_field_matcher={}, + file=output_file, + exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"], + ) + == number_of_assertions - 1 + ) diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json new file mode 100644 index 00000000000000..53688610dd0ce5 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json @@ -0,0 +1,1387 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831648843", + "id": "de711767-c7b9-4c33-99d7-510978dc1fa5", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "my_table_no_name", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_table_no_name", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"my-platform\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"my-platform\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:189046201d696e7810132cfa64dad337\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"my-platform\", \"folder_abs_path\": \"tests/integration/delta_lake\"}, \"name\": \"delta_lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:acf0f3806f475a7397ee745329ef2967\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"my-platform\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:1876d057d0ee364677b85427342e2c82\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"my-platform\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data/delta_tables\"}, \"name\": \"delta_tables\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:7888b6dab77b7e77709699c9a1b81aa4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a282913be26fceff334523c2be119df1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649166}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649715}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649731}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649754}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649788}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "3", + "partition_columns": "[]", + "table_creation_time": "1655664813952", + "id": "eca9d2a0-4ce6-4ace-a732-75fda0157fb8", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "my_table", + "qualifiedName": null, + "description": "my table description", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_table", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "day", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "month", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "sale_id", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "total_cost", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "float", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "year", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a282913be26fceff334523c2be119df1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"CONVERT\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655664815399}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831476360", + "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "test-table-basic", + "qualifiedName": null, + "description": "test delta table basic with table name", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-table-basic", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a282913be26fceff334523c2be119df1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831864836", + "id": "3775a0fd-4f58-4dea-b71a-e3fedb10f5b4", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "test-table-inner", + "qualifiedName": null, + "description": "test delta table basic with table name at inner location", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-table-inner", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"my-platform\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data/delta_tables/level1\"}, \"name\": \"level1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a282913be26fceff334523c2be119df1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831865396}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866337}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866398}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866447}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866541}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json new file mode 100644 index 00000000000000..161d97f2d37f68 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json @@ -0,0 +1,1387 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831648843", + "id": "de711767-c7b9-4c33-99d7-510978dc1fa5", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "my_table_no_name", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_table_no_name", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bdfaaacd66870755e65612e0b88dd4bf\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake\"}, \"name\": \"delta_lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:974a39dc631803eddedc699cc9bb9759\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:dae543a1ed7ecfea4079a971dc7805a6\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data/delta_tables\"}, \"name\": \"delta_tables\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ee050cda8eca59687021c24cbc0bb8a4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ad4b596846e8e010114b1ec82b324fab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649166}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649715}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649731}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649754}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649788}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "3", + "partition_columns": "[]", + "table_creation_time": "1655664813952", + "id": "eca9d2a0-4ce6-4ace-a732-75fda0157fb8", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "my_table", + "qualifiedName": null, + "description": "my table description", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_table", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "day", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "month", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "sale_id", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "total_cost", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "float", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + }, + { + "fieldPath": "year", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ad4b596846e8e010114b1ec82b324fab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"CONVERT\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655664815399}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831476360", + "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "test-table-basic", + "qualifiedName": null, + "description": "test delta table basic with table name", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-table-basic", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ad4b596846e8e010114b1ec82b324fab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831864836", + "id": "3775a0fd-4f58-4dea-b71a-e3fedb10f5b4", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables" + }, + "externalUrl": null, + "name": "test-table-inner", + "qualifiedName": null, + "description": "test delta table basic with table name at inner location", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-table-inner", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data/delta_tables/level1\"}, \"name\": \"level1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ad4b596846e8e010114b1ec82b324fab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:6bb6dc6de93177210067d00b45b481bb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831865396}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866337}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866398}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866447}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866541}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json new file mode 100644 index 00000000000000..8e9cb143ea9dd4 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json @@ -0,0 +1,300 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831476360", + "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic/" + }, + "externalUrl": null, + "name": "test-table-basic", + "qualifiedName": null, + "description": "test delta table basic with table name", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-table-basic", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"delta_tables\"}, \"name\": \"delta_tables\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:85267d161e1a2ffa647cec6c1188549f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json new file mode 100644 index 00000000000000..d48cd3c64cc814 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json @@ -0,0 +1,528 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "5", + "partition_columns": "['foo', 'bar']", + "table_creation_time": "1655831476360", + "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", + "version": "4", + "location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic" + }, + "externalUrl": null, + "name": "test-table-basic", + "qualifiedName": null, + "description": "test delta table basic with table name", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-table-basic", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 4, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "bar", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "foo", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": true, + "jsonProps": null + }, + { + "fieldPath": "zip", + "jsonPath": null, + "nullable": true, + "description": "{}", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bdfaaacd66870755e65612e0b88dd4bf\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake\"}, \"name\": \"delta_lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:974a39dc631803eddedc699cc9bb9759\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:dae543a1ed7ecfea4079a971dc7805a6\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"delta-lake\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/delta_lake/test_data/delta_tables\"}, \"name\": \"delta_tables\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:delta-lake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ee050cda8eca59687021c24cbc0bb8a4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ad4b596846e8e010114b1ec82b324fab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json new file mode 100644 index 00000000000000..ec3d4ea2f5769d --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json @@ -0,0 +1,12 @@ +{ + "type": "delta-lake", + "config": { + "env": "UAT", + "base_path": "tests/integration/delta_lake/test_data/delta_tables", + "platform_instance": "my-platform", + "table_pattern": { + "allow": ["s*"] + }, + "version_history_lookback": -1 + } +} diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json new file mode 100644 index 00000000000000..c7475b2d6e7285 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json @@ -0,0 +1,8 @@ +{ + "type": "delta-lake", + "config": { + "env": "UAT", + "base_path": "tests/integration/delta_lake/test_data/delta_tables", + "version_history_lookback": -1 + } +} diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json new file mode 100644 index 00000000000000..ffd736baa814ba --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json @@ -0,0 +1,9 @@ +{ + "type": "delta-lake", + "config": { + "env": "UAT", + "base_path": "tests/integration/delta_lake/test_data/", + "relative_path":"delta_tables/my_table_basic/", + "version_history_lookback": -1 + } +} diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json new file mode 100644 index 00000000000000..27fa25e09e099f --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json @@ -0,0 +1,7 @@ +{ + "type": "delta-lake", + "config": { + "env": "UAT", + "base_path": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic" + } +} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..c344027562aa8a --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1655724801453,"operation":"WRITE","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"468553a9-3b52-4e50-b855-e64d2d070cb8","name":"test-table","description":"test delta table","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"foo\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bar\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zip\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["foo","bar"],"configuration":{},"createdTime":1655724801057}} +{"add":{"path":"0","partitionValues":{"bar":"0","foo":"0"},"size":100,"modificationTime":1655724801375,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..181b31f71db52d --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655724802209,"operation":"WRITE","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"1","partitionValues":{"bar":"1","foo":"1"},"size":100,"modificationTime":1655724802208,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..007ed0efdd831b --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655724802227,"operation":"WRITE","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"2","partitionValues":{"bar":"0","foo":"2"},"size":100,"modificationTime":1655724802226,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json new file mode 100644 index 00000000000000..8ed5a1c9ec6b15 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000003.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655724802244,"operation":"WRITE","operationParameters":{},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"3","partitionValues":{"bar":"1","foo":"0"},"size":100,"modificationTime":1655724802243,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json new file mode 100644 index 00000000000000..23423b22f9fdcf --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test-table/_delta_log/00000000000000000004.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655724802264,"operation":"WRITE","operationParameters":{},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"4","partitionValues":{"bar":"0","foo":"1"},"size":100,"modificationTime":1655724802263,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..f93204d086bdc8 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1655831865396,"operation":"WRITE","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"3775a0fd-4f58-4dea-b71a-e3fedb10f5b4","name":"test-table-inner","description":"test delta table basic with table name at inner location","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"foo\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bar\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zip\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["foo","bar"],"configuration":{},"createdTime":1655831864836}} +{"add":{"path":"0","partitionValues":{"bar":"0","foo":"0"},"size":100,"modificationTime":1655831865289,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000001.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..80df25b214793e --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831866337,"operation":"WRITE","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"1","partitionValues":{"bar":"1","foo":"1"},"size":100,"modificationTime":1655831866336,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000002.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..936328a405e294 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831866398,"operation":"WRITE","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"2","partitionValues":{"bar":"0","foo":"2"},"size":100,"modificationTime":1655831866397,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000003.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000003.json new file mode 100644 index 00000000000000..ea66495b76f64f --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000003.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831866447,"operation":"WRITE","operationParameters":{},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"3","partitionValues":{"bar":"1","foo":"0"},"size":100,"modificationTime":1655831866446,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000004.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000004.json new file mode 100644 index 00000000000000..72b451d92fac70 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner/_delta_log/00000000000000000004.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831866541,"operation":"WRITE","operationParameters":{},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"4","partitionValues":{"bar":"0","foo":"1"},"size":100,"modificationTime":1655831866541,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..e913b115556d51 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1655831476907,"operation":"WRITE","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"628d06df-ecb0-4314-a97e-75d8872db7c3","name":"test-table-basic","description":"test delta table basic with table name","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"foo\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bar\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zip\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["foo","bar"],"configuration":{},"createdTime":1655831476360}} +{"add":{"path":"0","partitionValues":{"bar":"0","foo":"0"},"size":100,"modificationTime":1655831476810,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000001.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..adf1ac30f5ec23 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831477701,"operation":"WRITE","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"1","partitionValues":{"bar":"1","foo":"1"},"size":100,"modificationTime":1655831477701,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000002.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..9c2c7935fd05ab --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831477726,"operation":"WRITE","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"2","partitionValues":{"bar":"0","foo":"2"},"size":100,"modificationTime":1655831477725,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000003.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000003.json new file mode 100644 index 00000000000000..b3f11d22d01ea2 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000003.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831477745,"operation":"WRITE","operationParameters":{},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"3","partitionValues":{"bar":"1","foo":"0"},"size":100,"modificationTime":1655831477745,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000004.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000004.json new file mode 100644 index 00000000000000..f49ef6eb938d57 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_basic/_delta_log/00000000000000000004.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831477768,"operation":"WRITE","operationParameters":{},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"4","partitionValues":{"bar":"0","foo":"1"},"size":100,"modificationTime":1655831477768,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..16aaa1eb8f773b --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1655831649166,"operation":"WRITE","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"de711767-c7b9-4c33-99d7-510978dc1fa5","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"foo\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bar\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zip\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["foo","bar"],"configuration":{},"createdTime":1655831648843}} +{"add":{"path":"0","partitionValues":{"bar":"0","foo":"0"},"size":100,"modificationTime":1655831649141,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000001.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000001.json new file mode 100644 index 00000000000000..023dec3c38175b --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831649715,"operation":"WRITE","operationParameters":{},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"1","partitionValues":{"bar":"1","foo":"1"},"size":100,"modificationTime":1655831649715,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000002.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000002.json new file mode 100644 index 00000000000000..ef50652bfd336c --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831649731,"operation":"WRITE","operationParameters":{},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"2","partitionValues":{"bar":"0","foo":"2"},"size":100,"modificationTime":1655831649731,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000003.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000003.json new file mode 100644 index 00000000000000..abaefe3d71b136 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000003.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831649754,"operation":"WRITE","operationParameters":{},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"3","partitionValues":{"bar":"1","foo":"0"},"size":100,"modificationTime":1655831649754,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000004.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000004.json new file mode 100644 index 00000000000000..238ede0b54d7d6 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/my_table_no_name/_delta_log/00000000000000000004.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1655831649788,"operation":"WRITE","operationParameters":{},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"add":{"path":"4","partitionValues":{"bar":"0","foo":"1"},"size":100,"modificationTime":1655831649787,"dataChange":true,"tags":{"someTagKey":"someTagVal"}}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/._SUCCESS.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/._SUCCESS.crc new file mode 100644 index 00000000000000..3b7b044936a890 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/._SUCCESS.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet.crc new file mode 100644 index 00000000000000..193fbb5caeffe5 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet.crc new file mode 100644 index 00000000000000..833d4a9e30bef4 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet.crc new file mode 100644 index 00000000000000..bf059b2f012c53 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/.part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/_SUCCESS b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/_SUCCESS new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..c85c293b2fd9fe --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/_delta_log/00000000000000000000.json @@ -0,0 +1,6 @@ +{"commitInfo":{"timestamp":1655664815399,"operation":"CONVERT","operationParameters":{},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"local Delta-Standalone/0.4.0"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"eca9d2a0-4ce6-4ace-a732-75fda0157fb8","name":"my_table","description":"my table description","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"year\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"month\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"day\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"sale_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"customer\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"total_cost\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1655664813952}} +{"add":{"path":"part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet","partitionValues":{},"size":318922,"modificationTime":1654667918316,"dataChange":true}} +{"add":{"path":"part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet","partitionValues":{},"size":319296,"modificationTime":1654667918332,"dataChange":true}} +{"add":{"path":"part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet","partitionValues":{},"size":318646,"modificationTime":1654667918336,"dataChange":true}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet new file mode 100644 index 00000000000000..67353937c0663d Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-64c688b3-46cc-44c9-86a4-d1f07a3570c1-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet new file mode 100644 index 00000000000000..0219ea07b4cbdc Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-71b51e51-8746-425f-8e20-41dc771a1b47-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet new file mode 100644 index 00000000000000..802bc008a59c00 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables/sales/part-00000-c4f4550a-83d5-4c35-bdb6-9f8bd1a9d154-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_local_delta_lake.py b/metadata-ingestion/tests/integration/delta_lake/test_local_delta_lake.py new file mode 100644 index 00000000000000..412ea48614e2c3 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_local_delta_lake.py @@ -0,0 +1,76 @@ +import json +import logging +import os +import sys + +import pytest + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + +FROZEN_TIME = "2020-04-14 07:00:00" + +SOURCE_FILES_PATH = "./tests/integration/delta_lake/sources/local" +source_files = os.listdir(SOURCE_FILES_PATH) + + +@pytest.mark.parametrize("source_file", source_files) +@pytest.mark.skipif(sys.version_info < (3, 7), reason="delta-lake requires Python 3.7+") +def test_delta_lake(pytestconfig, source_file, tmp_path, mock_time): + test_resources_dir = pytestconfig.rootpath / "tests/integration/delta_lake" + + f = open(os.path.join(SOURCE_FILES_PATH, source_file)) + source = json.load(f) + + config_dict = {} + config_dict["source"] = source + config_dict["sink"] = { + "type": "file", + "config": { + "filename": f"{tmp_path}/{source_file}", + }, + # "type": "datahub-rest", + # "config": {"server": "http://localhost:8080"} + } + + config_dict["run_id"] = source_file + + pipeline = Pipeline.create(config_dict) + pipeline.run() + pipeline.raise_from_status() + + print(f"tmp pth: {tmp_path}") + print(f"source file : {source_file}") + print(f"testresource dir: {test_resources_dir}") + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/{source_file}", + golden_path=f"{test_resources_dir}/golden_files/local/golden_mces_{source_file}", + ) + + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="delta-lake requires Python 3.7+") +def test_data_lake_incorrect_config_raises_error(tmp_path, mock_time): + config_dict = {} + config_dict["sink"] = { + "type": "file", + "config": { + "filename": f"{tmp_path}/mces.json", + }, + # "type": "datahub-rest", + # "config": {"server": "http://localhost:8080"} + } + + # Case 1 : named variable in table name is not present in include + source = { + "type": "delta-lake", + "config": {"base_path": "invalid/path"}, + } + config_dict["source"] = source + with pytest.raises(Exception) as e_info: + pipeline = Pipeline.create(config_dict) + pipeline.run() + pipeline.raise_from_status() + + logging.debug(e_info) diff --git a/metadata-ingestion/tests/integration/feast-legacy/test_feast.py b/metadata-ingestion/tests/integration/feast-legacy/test_feast.py index 2f8a9abe04c52a..ce55ca336f1494 100644 --- a/metadata-ingestion/tests/integration/feast-legacy/test_feast.py +++ b/metadata-ingestion/tests/integration/feast-legacy/test_feast.py @@ -13,7 +13,7 @@ # make sure that mock_time is excluded here because it messes with feast @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path): test_resources_dir = pytestconfig.rootpath / "tests/integration/feast-legacy" diff --git a/metadata-ingestion/tests/integration/hana/hana_to_file.yml b/metadata-ingestion/tests/integration/hana/hana_to_file.yml index c37f37b884bc6b..8900a5a3bfbadf 100644 --- a/metadata-ingestion/tests/integration/hana/hana_to_file.yml +++ b/metadata-ingestion/tests/integration/hana/hana_to_file.yml @@ -32,7 +32,7 @@ source: include_field_histogram: true include_field_sample_values: true domain: - sales: + "urn:li:domain:sales": allow: - "HOTEL" sink: diff --git a/metadata-ingestion/tests/integration/hana/test_hana.py b/metadata-ingestion/tests/integration/hana/test_hana.py index 8982c86ddc2bad..0fa234d059e5eb 100644 --- a/metadata-ingestion/tests/integration/hana/test_hana.py +++ b/metadata-ingestion/tests/integration/hana/test_hana.py @@ -11,6 +11,7 @@ @freeze_time(FROZEN_TIME) +@pytest.mark.xfail # TODO: debug the flakes for this test @pytest.mark.slow_integration @pytest.mark.skipif( platform.machine().lower() == "aarch64", diff --git a/metadata-ingestion/tests/integration/hive/test_hive.py b/metadata-ingestion/tests/integration/hive/test_hive.py index d24f6c7de0da53..89853cf7bfe6cd 100644 --- a/metadata-ingestion/tests/integration/hive/test_hive.py +++ b/metadata-ingestion/tests/integration/hive/test_hive.py @@ -55,7 +55,7 @@ def base_pipeline_config(events_file): @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_hive_ingest( loaded_hive, pytestconfig, test_resources_dir, tmp_path, mock_time ): @@ -85,7 +85,7 @@ def test_hive_ingest( @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_hive_instance_check(loaded_hive, test_resources_dir, tmp_path, pytestconfig): instance: str = "production_warehouse" diff --git a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py index 2ae90a626308e9..56ea3839435e73 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py +++ b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py @@ -1,3 +1,4 @@ +import subprocess import time import pytest @@ -11,8 +12,19 @@ FROZEN_TIME = "2021-10-25 13:00:00" +def is_mysql_up(container_name: str, port: int) -> bool: + """A cheap way to figure out if mysql is responsive on a container""" + + cmd = f"docker logs {container_name} 2>&1 | grep '/var/run/mysqld/mysqld.sock' | grep {port}" + ret = subprocess.run( + cmd, + shell=True, + ) + return ret.returncode == 0 + + @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_kafka_connect_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/kafka-connect" test_resources_dir_kafka = pytestconfig.rootpath / "tests/integration/kafka" @@ -24,6 +36,13 @@ def test_kafka_connect_ingest(docker_compose_runner, pytestconfig, tmp_path, moc str(test_resources_dir / "docker-compose.override.yml"), ] with docker_compose_runner(docker_compose_file, "kafka-connect") as docker_services: + wait_for_port( + docker_services, + "test_mysql", + 3306, + timeout=120, + checker=lambda: is_mysql_up("test_mysql", 3306), + ) wait_for_port(docker_services, "test_broker", 59092, timeout=120) wait_for_port(docker_services, "test_connect", 58083, timeout=120) docker_services.wait_until_responsive( diff --git a/metadata-ingestion/tests/integration/kafka/docker-compose.yml b/metadata-ingestion/tests/integration/kafka/docker-compose.yml index 136438ab468689..4f00dd1ea884b0 100644 --- a/metadata-ingestion/tests/integration/kafka/docker-compose.yml +++ b/metadata-ingestion/tests/integration/kafka/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: zookeeper: - image: confluentinc/cp-zookeeper:5.4.0 + image: confluentinc/cp-zookeeper:7.2.0 env_file: zookeeper.env hostname: test_zookeeper ports: @@ -11,7 +11,7 @@ services: - test_zkdata:/var/opt/zookeeper broker: - image: confluentinc/cp-kafka:5.4.0 + image: confluentinc/cp-kafka:7.2.0 env_file: broker.env hostname: test_broker container_name: test_broker @@ -21,7 +21,7 @@ services: - "59092:59092" schema-registry: - image: confluentinc/cp-schema-registry:5.4.0 + image: confluentinc/cp-schema-registry:7.2.0 env_file: schema-registry.env container_name: test_schema_registry depends_on: diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml index 69c43c57aa44bd..b345ba86799371 100644 --- a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml +++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml @@ -7,7 +7,7 @@ source: bootstrap: "localhost:59092" schema_registry_url: "http://localhost:58081" domain: - sales: + "urn:li:domain:sales": allow: - "key_value_topic" sink: diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json index de8ef316cdc525..c88815bfba63f0 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json @@ -10,7 +10,10 @@ "displayName": null, "email": "simpons-group", "admins": [], - "members": [], + "members": [ + "urn:li:corpuser:hsimpson", + "urn:li:corpuser:lsimpson" + ], "groups": [], "description": null } @@ -69,7 +72,7 @@ "email": "hsimpson", "title": "Mr. Everything", "managerUrn": "urn:li:corpuser:bsimpson", - "departmentId": null, + "departmentId": 1001, "departmentName": "1001", "firstName": "Homer", "lastName": "Simpson", @@ -148,5 +151,93 @@ "runId": "ldap-test", "properties": null } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:hbevan", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Hester Bevan", + "email": "hbevan", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Hester", + "lastName": "Bevan", + "fullName": "Hester Bevan", + "countryCode": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test", + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:ehaas", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Evalyn Haas", + "email": "ehaas", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Evalyn", + "lastName": "Haas", + "fullName": "Evalyn Haas", + "countryCode": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test", + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:HR Department", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "displayName": null, + "email": "HR Department", + "admins": [], + "members": [], + "groups": [], + "description": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test", + "properties": null + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json new file mode 100644 index 00000000000000..1ac0c691f3e908 --- /dev/null +++ b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json @@ -0,0 +1,82 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:hbevan", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Hester Bevan", + "email": "hbevan", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Hester", + "lastName": "Bevan", + "fullName": "Hester Bevan", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:HR Department" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:ehaas", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Evalyn Haas", + "email": "ehaas", + "title": null, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": "Evalyn", + "lastName": "Haas", + "fullName": "Evalyn Haas", + "countryCode": null + } + }, + { + "com.linkedin.pegasus2avro.identity.GroupMembership": { + "groups": [ + "urn:li:corpGroup:HR Department" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif b/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif index 049507ea3364dd..ecb39d64ea6a77 100644 --- a/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif +++ b/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif @@ -9,6 +9,19 @@ version: 1 +# Enable memberOf overlay +dn: olcOverlay=memberof,olcDatabase={1}mdb,cn=config +objectClass: olcMemberOf +objectClass: olcOverlayConfig +objectClass: olcConfig +objectClass: top +olcOverlay: memberof +olcMemberOfDangling: ignore +olcMemberOfRefInt: TRUE +olcMemberOfGroupOC: groupOfNames +olcMemberOfMemberAD: member +olcMemberOfMemberOfAD: memberOf + # Entry 1: dc=example,dc=org # Note: this is commented out because the containers bootstrap scripts already # handle this for us. @@ -35,6 +48,8 @@ cn: simpons-group gidnumber: 500 objectclass: posixGroup objectclass: top +memberUid: hsimpson +memberUid: lsimpson # Entry 4: ou=people,dc=example,dc=org dn: ou=people,dc=example,dc=org @@ -111,3 +126,39 @@ dn: ou=Sales Department,dc=example,dc=org objectclass: organizationalUnit objectclass: top ou: Sales Department + +# Entry 10: cn=Hester Bevan,ou=people,dc=example,dc=org +dn: cn=Hester Bevan,ou=people,dc=example,dc=org +cn: Hester Bevan +gidnumber: 500 +givenname: Hester +homedirectory: /home/users/hbevan +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: Bevan +uid: hbevan +uidnumber: 1004 +userpassword: {MD5}4QrcOUm6Wau+VuBX8g+IPg== + +# Entry 11: cn=Evalyn Haas,ou=people,dc=example,dc=org +dn: cn=Evalyn Haas,ou=people,dc=example,dc=org +cn: Evalyn Haas +gidnumber: 500 +givenname: Evalyn +homedirectory: /home/users/ehaas +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: Haas +uid: ehaas +uidnumber: 1005 +userpassword: {MD5}4QrcOUm6Wau+VuBX8g+IPg== + +# Entry 12: cn=HR Department,ou=groups,dc=example,dc=org +dn: cn=HR Department,dc=example,dc=org +cn: HR Department +objectclass: groupOfNames +objectclass: top +member: cn=Hester Bevan,ou=people,dc=example,dc=org +member: cn=Evalyn Haas,ou=people,dc=example,dc=org \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/ldap/test_ldap.py b/metadata-ingestion/tests/integration/ldap/test_ldap.py index 94cef8bed548d0..38992388873b64 100644 --- a/metadata-ingestion/tests/integration/ldap/test_ldap.py +++ b/metadata-ingestion/tests/integration/ldap/test_ldap.py @@ -29,6 +29,9 @@ def test_ldap_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): "ldap_user": "cn=admin,dc=example,dc=org", "ldap_password": "admin", "base_dn": "dc=example,dc=org", + "group_attrs_map": { + "members": "memberUid", + }, }, }, "sink": { @@ -47,3 +50,50 @@ def test_ldap_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): output_path=tmp_path / "ldap_mces.json", golden_path=test_resources_dir / "ldap_mces_golden.json", ) + + +@pytest.mark.integration +def test_ldap_memberof_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): + test_resources_dir = pytestconfig.rootpath / "tests/integration/ldap" + + with docker_compose_runner( + test_resources_dir / "docker-compose.yml", "ldap" + ) as docker_services: + # The openldap container loads the sample data after exposing the port publicly. As such, + # we must wait a little bit extra to ensure that the sample data is loaded. + wait_for_port(docker_services, "openldap", 389) + time.sleep(5) + + pipeline = Pipeline.create( + { + "run_id": "ldap-test", + "source": { + "type": "ldap", + "config": { + "ldap_server": "ldap://localhost", + "ldap_user": "cn=admin,dc=example,dc=org", + "ldap_password": "admin", + "base_dn": "dc=example,dc=org", + "filter": "(memberOf=cn=HR Department,dc=example,dc=org)", + "attrs_list": ["+", "*"], + "group_attrs_map": { + "members": "member", + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/ldap_memberof_mces.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "ldap_memberof_mces.json", + golden_path=test_resources_dir / "ldap_memberof_mces_golden.json", + ) diff --git a/metadata-ingestion/tests/integration/looker/expected_output.json b/metadata-ingestion/tests/integration/looker/expected_output.json index f4e272e04ceac1..96393ce4b531a3 100644 --- a/metadata-ingestion/tests/integration/looker/expected_output.json +++ b/metadata-ingestion/tests/integration/looker/expected_output.json @@ -56,6 +56,7 @@ "charts": [ "urn:li:chart:(looker,dashboard_elements.2)" ], + "datasets": [], "lastModified": { "created": { "time": 0, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json index 044dbbd948e012..df0190b8e72984 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json @@ -12,18 +12,17 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null }, @@ -88,8 +87,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.underlying_view,PROD)", "type": "VIEW" @@ -106,14 +104,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -142,7 +138,8 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:Dimension" + "tag": "urn:li:tag:Dimension", + "context": null } ] }, @@ -206,8 +203,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -248,8 +244,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -290,8 +285,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json index 044dbbd948e012..df0190b8e72984 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json @@ -12,18 +12,17 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null }, @@ -88,8 +87,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.underlying_view,PROD)", "type": "VIEW" @@ -106,14 +104,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -142,7 +138,8 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:Dimension" + "tag": "urn:li:tag:Dimension", + "context": null } ] }, @@ -206,8 +203,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -248,8 +244,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -290,8 +285,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json index 5a6204407feb40..35bf69572d2a15 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json @@ -12,18 +12,17 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null }, @@ -88,8 +87,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_joined_view,PROD)", "type": "VIEW" @@ -98,8 +96,16 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_joined_view_original_name,PROD)", + "type": "VIEW" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view_has_no_fields,PROD)", "type": "VIEW" @@ -108,8 +114,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.underlying_view,PROD)", "type": "VIEW" @@ -126,14 +131,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -162,7 +165,8 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:Dimension" + "tag": "urn:li:tag:Dimension", + "context": null } ] }, @@ -226,8 +230,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -268,8 +271,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -310,8 +312,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json index 3a835d95845168..4438c07bb0ad46 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json @@ -12,18 +12,17 @@ "title": "foo", "description": "lorem ipsum", "charts": [], + "datasets": [], "lastModified": { "created": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { - "time": 0, + "time": 1586847600000, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null }, @@ -88,8 +87,16 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_joined_view_original_name,PROD)", + "type": "VIEW" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", "type": "VIEW" @@ -98,8 +105,7 @@ "auditStamp": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view_has_no_fields,PROD)", "type": "VIEW" @@ -116,14 +122,12 @@ "created": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null }, "deleted": null, "dataset": null, @@ -152,7 +156,8 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:Dimension" + "tag": "urn:li:tag:Dimension", + "context": null } ] }, @@ -216,8 +221,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -258,8 +262,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, @@ -300,8 +303,7 @@ "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown", - "impersonator": null, - "message": null + "impersonator": null } } }, diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json new file mode 100644 index 00000000000000..829486509af34a --- /dev/null +++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json @@ -0,0 +1,387 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(looker,dashboards.1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "externalUrl": null, + "title": "foo", + "description": "lorem ipsum", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1586847600000, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 1586847600000, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null + }, + "dashboardUrl": "https://looker.company.com/dashboards/1", + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.1)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"viewsCount\": 25, \"favoritesCount\": 5, \"lastViewedAt\": 1586847600000}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/looker/lkml_samples/explores/data.my_view" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.explore.label": "My Explore View", + "looker.explore.file": "test_source_file.lkml" + }, + "externalUrl": "https://looker.company.com/explore/data/my_view", + "name": "My Explore View", + "qualifiedName": null, + "description": "lorem ipsum", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.underlying_view,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_view", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "dim1", + "jsonPath": null, + "nullable": false, + "description": "", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": [], + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,data.explore.my_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"explore\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Dimension", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Dimension", + "description": "A tag that is applied to all dimension fields.", + "colorHex": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Temporal", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Temporal", + "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations.", + "colorHex": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Measure", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Measure", + "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on", + "colorHex": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.11)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1656979200000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"executionsCount\": 14, \"uniqueUserCount\": 1, \"userCounts\": [{\"user\": \"urn:li:corpuser:test@looker.com\", \"executionsCount\": 14, \"usageCount\": 14, \"userEmail\": \"test@looker.com\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.12)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1656979200000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"executionsCount\": 14, \"uniqueUserCount\": 1, \"userCounts\": [{\"user\": \"urn:li:corpuser:test@looker.com\", \"executionsCount\": 14, \"usageCount\": 14, \"userEmail\": \"test@looker.com\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(looker,dashboards.37)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "value": "{\"timestampMillis\": 1656979200000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"executionsCount\": 5, \"uniqueUserCount\": 1, \"userCounts\": [{\"user\": \"urn:li:corpuser:test@looker.com\", \"executionsCount\": 5, \"usageCount\": 5, \"userEmail\": \"test@looker.com\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "looker-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index c89c7a12211bf1..53e2ec875f6657 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -1,8 +1,11 @@ +import json import time from datetime import datetime +from typing import Optional from unittest import mock from freezegun import freeze_time +from looker_sdk.rtl.transport import TransportOptions from looker_sdk.sdk.api31.models import ( Dashboard, DashboardElement, @@ -11,9 +14,12 @@ LookmlModelExploreFieldset, LookmlModelExploreJoins, Query, + User, + WriteQuery, ) from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.looker import usage_queries from tests.test_helpers import mce_helpers FROZEN_TIME = "2020-04-14 07:00:00" @@ -29,6 +35,7 @@ def test_looker_ingest(pytestconfig, tmp_path, mock_time): id="1", title="foo", created_at=datetime.utcfromtimestamp(time.time()), + updated_at=datetime.utcfromtimestamp(time.time()), description="lorem ipsum", dashboard_elements=[ DashboardElement( @@ -87,6 +94,7 @@ def test_looker_ingest_joins(pytestconfig, tmp_path, mock_time): id="1", title="foo", created_at=datetime.utcfromtimestamp(time.time()), + updated_at=datetime.utcfromtimestamp(time.time()), description="lorem ipsum", dashboard_elements=[ DashboardElement( @@ -145,6 +153,7 @@ def test_looker_ingest_unaliased_joins(pytestconfig, tmp_path, mock_time): id="1", title="foo", created_at=datetime.utcfromtimestamp(time.time()), + updated_at=datetime.utcfromtimestamp(time.time()), description="lorem ipsum", dashboard_elements=[ DashboardElement( @@ -222,6 +231,10 @@ def setup_mock_explore_with_joins(mocked_client): relationship="one_to_one", sql_on="1=1", ), + LookmlModelExploreJoins( + name="my_joined_view_join_name", + from_="my_joined_view_original_name", + ), ], ) @@ -249,7 +262,11 @@ def setup_mock_explore_unaliased_with_joins(mocked_client): view_label="My Labeled View", relationship="one_to_one", sql_on="1=1", - ) + ), + LookmlModelExploreJoins( + name="my_joined_view_join_name", + from_="my_joined_view_original_name", + ), ], ) @@ -275,6 +292,71 @@ def setup_mock_explore(mocked_client): ) +def setup_mock_user(mocked_client): + mocked_client.user.return_value = User(id=1, email="test@looker.com") + + +def side_effect_query_inline( + result_format: str, body: WriteQuery, transport_options: Optional[TransportOptions] +) -> str: + query_type = None + if result_format == "sql": + return "" # Placeholder for sql text + for query_name, query_template in usage_queries.items(): + if body.fields == query_template["fields"]: + query_type = query_name + + if query_type == "counts_per_day_per_dashboard": + return json.dumps( + [ + { + "history.dashboard_id": "11", + "history.created_date": "2022-07-05", + "history.dashboard_user": 1, + "history.dashboard_run_count": 14, + }, + { + "history.dashboard_id": "12", + "history.created_date": "2022-07-05", + "history.dashboard_user": 1, + "history.dashboard_run_count": 14, + }, + { + "history.dashboard_id": "37", + "history.created_date": "2022-07-05", + "history.dashboard_user": 1, + "history.dashboard_run_count": 5, + }, + ] + ) + + if query_type == "counts_per_day_per_user_per_dashboard": + return json.dumps( + [ + { + "history.created_date": "2022-07-05", + "history.dashboard_id": "11", + "user.id": 1, + "history.dashboard_run_count": 14, + }, + { + "history.created_date": "2022-07-05", + "history.dashboard_id": "12", + "user.id": 1, + "history.dashboard_run_count": 14, + }, + { + "history.created_date": "2022-07-05", + "history.dashboard_id": "37", + "user.id": 1, + "history.dashboard_run_count": 5, + }, + ] + ) + + raise Exception("Unknown Query") + + @freeze_time(FROZEN_TIME) def test_looker_ingest_allow_pattern(pytestconfig, tmp_path, mock_time): mocked_client = mock.MagicMock() @@ -286,6 +368,7 @@ def test_looker_ingest_allow_pattern(pytestconfig, tmp_path, mock_time): id="1", title="foo", created_at=datetime.utcfromtimestamp(time.time()), + updated_at=datetime.utcfromtimestamp(time.time()), description="lorem ipsum", dashboard_elements=[ DashboardElement( @@ -343,3 +426,68 @@ def test_looker_ingest_allow_pattern(pytestconfig, tmp_path, mock_time): output_path=tmp_path / "looker_mces.json", golden_path=f"{test_resources_dir}/{mce_out_file}", ) + + +@freeze_time(FROZEN_TIME) +def test_looker_ingest_usage_history(pytestconfig, tmp_path, mock_time): + mocked_client = mock.MagicMock() + with mock.patch("looker_sdk.init31") as mock_sdk: + mock_sdk.return_value = mocked_client + mocked_client.all_dashboards.return_value = [Dashboard(id="1")] + mocked_client.dashboard.return_value = Dashboard( + id="1", + title="foo", + created_at=datetime.utcfromtimestamp(time.time()), + updated_at=datetime.utcfromtimestamp(time.time()), + description="lorem ipsum", + favorite_count=5, + view_count=25, + last_viewed_at=datetime.utcfromtimestamp(time.time()), + dashboard_elements=[ + DashboardElement( + id="2", + type="", + subtitle_text="Some text", + query=Query( + model="data", + view="my_view", + dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]', + ), + ) + ], + ) + mocked_client.run_inline_query.side_effect = side_effect_query_inline + setup_mock_explore(mocked_client) + setup_mock_user(mocked_client) + + test_resources_dir = pytestconfig.rootpath / "tests/integration/looker" + + pipeline = Pipeline.create( + { + "run_id": "looker-test", + "source": { + "type": "looker", + "config": { + "base_url": "https://looker.company.com", + "client_id": "foo", + "client_secret": "bar", + "extract_usage_history": True, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/looker_mces.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + mce_out_file = "looker_mces_usage_history.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "looker_mces.json", + golden_path=f"{test_resources_dir}/{mce_out_file}", + ) diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples/data.model.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples/data.model.lkml index f28fe84d155f87..f7336332647ce9 100644 --- a/metadata-ingestion/tests/integration/lookml/lkml_samples/data.model.lkml +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples/data.model.lkml @@ -30,4 +30,4 @@ explore: data_model { value: "day" } } -} +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples/data2.model.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples/data2.model.lkml new file mode 100644 index 00000000000000..b1cd88dffd526a --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples/data2.model.lkml @@ -0,0 +1,10 @@ +connection: "my_other_connection" +include: "**/*.view.lkml" + +explore: aliased_explore2 { + from: my_view2 +} + +explore: duplicate_explore { + from: my_view +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lkml_samples/foo2.view.lkml b/metadata-ingestion/tests/integration/lookml/lkml_samples/foo2.view.lkml new file mode 100644 index 00000000000000..6edbada8173d6c --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/lkml_samples/foo2.view.lkml @@ -0,0 +1,47 @@ +view: my_view2 { + derived_table: { + sql: + SELECT + is_latest, + country, + city, + timestamp, + measurement + FROM + my_table ;; + } + + dimension: country { + type: string + description: "The country" + sql: ${TABLE}.country ;; + } + + dimension: city { + type: string + description: "City" + sql: ${TABLE}.city ;; + } + + dimension: is_latest { + type: yesno + description: "Is latest data" + sql: ${TABLE}.is_latest ;; + } + + dimension_group: timestamp { + group_label: "Timestamp" + type: time + description: "Timestamp of measurement" + sql: ${TABLE}.timestamp ;; + timeframes: [hour, date, week, day_of_week] + } + + measure: average_measurement { + group_label: "Measurement" + type: average + description: "My measurement" + sql: ${TABLE}.measurement ;; + } + +} diff --git a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json new file mode 100644 index 00000000000000..d3e376c499c285 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json @@ -0,0 +1,659 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/looker/lkml_samples/views/my_view" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,warehouse.default_db.default_schema.my_table,DEV)", + "type": "VIEW" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_view", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "country", + "jsonPath": null, + "nullable": false, + "description": "The country", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "city", + "jsonPath": null, + "nullable": false, + "description": "City", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "is_latest", + "jsonPath": null, + "nullable": false, + "description": "Is latest data", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "yesno", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": "Timestamp of measurement", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "time", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + }, + { + "tag": "urn:li:tag:Temporal", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "average_measurement", + "jsonPath": null, + "nullable": false, + "description": "My measurement", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "average", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Measure", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": [], + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "foo.view.lkml" + }, + "externalUrl": null, + "name": "my_view", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"SELECT\\n is_latest,\\n country,\\n city,\\n timestamp,\\n measurement\\n FROM\\n my_table\", \"viewLanguage\": \"sql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/looker/lkml_samples/views/my_view2" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,rs_warehouse.default_db.default_schema.my_table,DEV)", + "type": "VIEW" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "my_view2", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "country", + "jsonPath": null, + "nullable": false, + "description": "The country", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "city", + "jsonPath": null, + "nullable": false, + "description": "City", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "is_latest", + "jsonPath": null, + "nullable": false, + "description": "Is latest data", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "yesno", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "timestamp", + "jsonPath": null, + "nullable": false, + "description": "Timestamp of measurement", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "time", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension", + "context": null + }, + { + "tag": "urn:li:tag:Temporal", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "average_measurement", + "jsonPath": null, + "nullable": false, + "description": "My measurement", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "average", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Measure", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": [], + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "foo2.view.lkml" + }, + "externalUrl": null, + "name": "my_view2", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"SELECT\\n is_latest,\\n country,\\n city,\\n timestamp,\\n measurement\\n FROM\\n my_table\", \"viewLanguage\": \"sql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Dimension", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Dimension", + "description": "A tag that is applied to all dimension fields.", + "colorHex": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Temporal", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Temporal", + "description": "A tag that is applied to all time-based (temporal) fields such as timestamps or durations.", + "colorHex": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Measure", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Measure", + "description": "A tag that is applied to all measures (metrics). Measures are typically the columns that you aggregate on", + "colorHex": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index da00d2748164b4..b4b352d5771b65 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -10,7 +10,7 @@ from datahub.configuration.common import PipelineExecutionError from datahub.ingestion.run.pipeline import Pipeline -from tests.test_helpers import mce_helpers # noqa: F401 +from tests.test_helpers import mce_helpers logging.getLogger("lkml").setLevel(logging.INFO) @@ -39,6 +39,7 @@ def test_lookml_ingest(pytestconfig, tmp_path, mock_time): "parse_table_names_from_sql": True, "tag_measures_and_dimensions": False, "project_name": "lkml_samples", + "model_pattern": {"deny": ["data2"]}, }, }, "sink": { @@ -82,6 +83,7 @@ def test_lookml_ingest_offline(pytestconfig, tmp_path, mock_time): }, "parse_table_names_from_sql": True, "project_name": "lkml_samples", + "model_pattern": {"deny": ["data2"]}, }, }, "sink": { @@ -171,6 +173,7 @@ def test_lookml_ingest_offline_platform_instance(pytestconfig, tmp_path, mock_ti }, "parse_table_names_from_sql": True, "project_name": "lkml_samples", + "model_pattern": {"deny": ["data2"]}, }, }, "sink": { @@ -226,7 +229,7 @@ def ingestion_test( tmp_path: pathlib.Path, mock_time: int, mock_connection: DBConnection, -) -> None: # noqa : No need for type annotations here +) -> None: test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" mce_out_file = f"lookml_mces_api_{mock_connection.dialect_name}.json" mocked_client = mock.MagicMock() @@ -250,6 +253,7 @@ def ingestion_test( "base_url": "fake_account.looker.com", }, "parse_table_names_from_sql": True, + "model_pattern": {"deny": ["data2"]}, }, }, "sink": { @@ -342,6 +346,7 @@ def test_lookml_github_info(pytestconfig, tmp_path, mock_time): }, "parse_table_names_from_sql": True, "project_name": "lkml_samples", + "model_pattern": {"deny": ["data2"]}, "github_info": {"repo": "datahub/looker-demo", "branch": "master"}, }, }, @@ -362,3 +367,68 @@ def test_lookml_github_info(pytestconfig, tmp_path, mock_time): output_path=tmp_path / mce_out, golden_path=test_resources_dir / mce_out, ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.skipif(sys.version_info < (3, 7), reason="lkml requires Python 3.7+") +def test_reachable_views(pytestconfig, tmp_path, mock_time): + """Test for reachable views""" + test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" + mce_out = "lookml_reachable_views.json" + pipeline = Pipeline.create( + { + "run_id": "lookml-test", + "source": { + "type": "lookml", + "config": { + "base_folder": str(test_resources_dir / "lkml_samples"), + "connection_to_platform_map": { + "my_connection": { + "platform": "snowflake", + "platform_instance": "warehouse", + "platform_env": "dev", + "default_db": "default_db", + "default_schema": "default_schema", + }, + "my_other_connection": { + "platform": "redshift", + "platform_instance": "rs_warehouse", + "platform_env": "dev", + "default_db": "default_db", + "default_schema": "default_schema", + }, + }, + "parse_table_names_from_sql": True, + "project_name": "lkml_samples", + "emit_reachable_views_only": True, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/{mce_out}", + }, + }, + } + ) + pipeline.run() + pipeline.pretty_print_summary() + pipeline.raise_from_status(raise_warnings=True) + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / mce_out, + golden_path=test_resources_dir / mce_out, + ) + + entity_urns = mce_helpers.get_entity_urns(tmp_path / mce_out) + # we should only have two views discoverable + assert len(entity_urns) == 2 + assert ( + "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view,PROD)" + in entity_urns + ) + assert ( + "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.my_view2,PROD)" + in entity_urns + ) diff --git a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json index 127988ba381d7b..3029f815e58bb8 100644 --- a/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json +++ b/metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json @@ -15,6 +15,7 @@ "urn:li:chart:(metabase,1)", "urn:li:chart:(metabase,2)" ], + "datasets": [], "lastModified": { "created": { "time": 1639417721742, diff --git a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json index 16e9bd0db56226..003a74ed0a6d12 100644 --- a/metadata-ingestion/tests/integration/mode/mode_mces_golden.json +++ b/metadata-ingestion/tests/integration/mode/mode_mces_golden.json @@ -14,6 +14,7 @@ "charts": [ "urn:li:chart:(mode,f622b9ee725b)" ], + "datasets": [], "lastModified": { "created": { "time": 1639169724316, diff --git a/metadata-ingestion/tests/integration/mysql/mysql_to_file.yml b/metadata-ingestion/tests/integration/mysql/mysql_to_file.yml index 988dbca18ff366..8d4a5b84b91d20 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_to_file.yml +++ b/metadata-ingestion/tests/integration/mysql/mysql_to_file.yml @@ -31,7 +31,7 @@ source: include_field_histogram: true include_field_sample_values: true domain: - sales: + "urn:li:domain:sales": allow: - "^metagalaxy" sink: diff --git a/metadata-ingestion/tests/integration/mysql/mysql_to_file_dbalias.yml b/metadata-ingestion/tests/integration/mysql/mysql_to_file_dbalias.yml index 86e5915a0d1d25..1c324641fe1583 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_to_file_dbalias.yml +++ b/metadata-ingestion/tests/integration/mysql/mysql_to_file_dbalias.yml @@ -32,7 +32,7 @@ source: include_field_histogram: true include_field_sample_values: true domain: - sales: + "urn:li:domain:sales": allow: - "^metagalaxy" sink: diff --git a/metadata-ingestion/tests/integration/mysql/test_mysql.py b/metadata-ingestion/tests/integration/mysql/test_mysql.py index 29f12f677e703b..9608152d05f8b6 100644 --- a/metadata-ingestion/tests/integration/mysql/test_mysql.py +++ b/metadata-ingestion/tests/integration/mysql/test_mysql.py @@ -1,3 +1,5 @@ +import subprocess + import pytest from freezegun import freeze_time @@ -6,60 +8,80 @@ from tests.test_helpers.docker_helpers import wait_for_port FROZEN_TIME = "2020-04-14 07:00:00" +MYSQL_PORT = 3306 -@freeze_time(FROZEN_TIME) -@pytest.mark.integration -def test_mysql_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): - test_resources_dir = pytestconfig.rootpath / "tests/integration/mysql" +@pytest.fixture(scope="module") +def test_resources_dir(pytestconfig): + return pytestconfig.rootpath / "tests/integration/mysql" + + +def is_mysql_up(container_name: str, port: int) -> bool: + """A cheap way to figure out if mysql is responsive on a container""" + + cmd = f"docker logs {container_name} 2>&1 | grep '/usr/sbin/mysqld: ready for connections.' | grep {port}" + ret = subprocess.run( + cmd, + shell=True, + ) + return ret.returncode == 0 + +@pytest.fixture(scope="module") +def mysql_runner(docker_compose_runner, pytestconfig, test_resources_dir): with docker_compose_runner( test_resources_dir / "docker-compose.yml", "mysql" ) as docker_services: - wait_for_port(docker_services, "testmysql", 3306) - - # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "mysql_to_file.yml").resolve() - run_datahub_cmd( - ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path - ) - - # Verify the output. - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / "mysql_mces.json", - golden_path=test_resources_dir / "mysql_mces_golden.json", + wait_for_port( + docker_services, + "testmysql", + MYSQL_PORT, + timeout=120, + checker=lambda: is_mysql_up("testmysql", MYSQL_PORT), ) + yield docker_services @freeze_time(FROZEN_TIME) @pytest.mark.integration -def test_mysql_ingest_with_db_alias( - docker_compose_runner, pytestconfig, tmp_path, mock_time +def test_mysql_ingest( + mysql_runner, pytestconfig, test_resources_dir, tmp_path, mock_time ): - test_resources_dir = pytestconfig.rootpath / "tests/integration/mysql" + # Run the metadata ingestion pipeline. + config_file = (test_resources_dir / "mysql_to_file.yml").resolve() + run_datahub_cmd( + ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path + ) - with docker_compose_runner( - test_resources_dir / "docker-compose.yml", "mysql" - ) as docker_services: - wait_for_port(docker_services, "testmysql", 3306) + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "mysql_mces.json", + golden_path=test_resources_dir / "mysql_mces_golden.json", + ) - # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "mysql_to_file_dbalias.yml").resolve() - run_datahub_cmd( - ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path - ) - # Verify the output. - # Assert that all events generated have instance specific urns - import re +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_mysql_ingest_with_db_alias( + mysql_runner, pytestconfig, test_resources_dir, tmp_path, mock_time +): + # Run the metadata ingestion pipeline. + config_file = (test_resources_dir / "mysql_to_file_dbalias.yml").resolve() + run_datahub_cmd( + ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path + ) - urn_pattern = "^" + re.escape( - "urn:li:dataset:(urn:li:dataPlatform:mysql,foogalaxy." - ) - mce_helpers.assert_mcp_entity_urn( - filter="ALL", - entity_type="dataset", - regex_pattern=urn_pattern, - file=tmp_path / "mysql_mces_dbalias.json", - ) + # Verify the output. + # Assert that all events generated have instance specific urns + import re + + urn_pattern = "^" + re.escape( + "urn:li:dataset:(urn:li:dataPlatform:mysql,foogalaxy." + ) + mce_helpers.assert_mcp_entity_urn( + filter="ALL", + entity_type="dataset", + regex_pattern=urn_pattern, + file=tmp_path / "mysql_mces_dbalias.json", + ) diff --git a/metadata-ingestion/tests/integration/okta/okta_mces_golden_custom_user_name_regex.json b/metadata-ingestion/tests/integration/okta/okta_mces_golden_custom_user_name_regex.json index 20d9ee9222cd93..eca7745d85e029 100644 --- a/metadata-ingestion/tests/integration/okta/okta_mces_golden_custom_user_name_regex.json +++ b/metadata-ingestion/tests/integration/okta/okta_mces_golden_custom_user_name_regex.json @@ -22,6 +22,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:All%20Employees", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -48,6 +69,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:Engineering", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -87,6 +129,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:john.doe@test.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -128,6 +191,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:mary.jane@test.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } } diff --git a/metadata-ingestion/tests/integration/okta/okta_mces_golden_default_config.json b/metadata-ingestion/tests/integration/okta/okta_mces_golden_default_config.json index f24d6770e4bd9a..9714b8df719b0c 100644 --- a/metadata-ingestion/tests/integration/okta/okta_mces_golden_default_config.json +++ b/metadata-ingestion/tests/integration/okta/okta_mces_golden_default_config.json @@ -22,6 +22,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:All%20Employees", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -48,6 +69,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:Engineering", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -87,6 +129,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:john.doe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -128,6 +191,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:mary.jane", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } } diff --git a/metadata-ingestion/tests/integration/okta/okta_mces_golden_include_deprovisioned_suspended_users.json b/metadata-ingestion/tests/integration/okta/okta_mces_golden_include_deprovisioned_suspended_users.json index 28353e8729f165..22e0ebf3aae173 100644 --- a/metadata-ingestion/tests/integration/okta/okta_mces_golden_include_deprovisioned_suspended_users.json +++ b/metadata-ingestion/tests/integration/okta/okta_mces_golden_include_deprovisioned_suspended_users.json @@ -22,6 +22,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:All%20Employees", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -48,6 +69,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:Engineering", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -87,6 +129,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:john.doe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -128,6 +191,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:mary.jane", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -169,6 +253,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:mary.jane", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -208,6 +313,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:bad.boyjones", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } }, @@ -247,6 +373,27 @@ "systemMetadata": { "lastObserved": 1586847600000, "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:bad.girlriri", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "origin", + "aspect": { + "value": "{\"type\": \"EXTERNAL\", \"externalType\": \"OKTA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "test-okta-usage", + "registryName": null, + "registryVersion": null, "properties": null } } diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json new file mode 100644 index 00000000000000..1d5fde3ec0a713 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -0,0 +1,173 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"description\": \"issue_history\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "value": "{\"customProperties\": {\"datasetId\": \"05169CD2-E713-41E6-9600-1D8066D95445\", \"reportId\": \"\", \"datasetWebUrl\": \"http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details\", \"createdFrom\": \"Dataset\"}, \"title\": \"test_tile\", \"description\": \"test_tile\", \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"inputs\": [{\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"chartId\": \"powerbi.linkedin.com/charts/B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "value": "{\"paths\": [\"/powerbi/64ED5CAD-7C10-4684-8180-826122881108\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"demo-workspace\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index f7f8f285c1c4cb..62ec08afd2f9a2 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1,382 +1,306 @@ [ - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_book,DEV)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"description\": \"issue_book\", \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"description\": \"issue_history\", \"tags\": []}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_book,DEV)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.member,DEV)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"description\": \"member\", \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.member,DEV)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "value": "{\"customProperties\": {}, \"description\": \"issue_history\", \"tags\": []}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "value": "{\"active\": true, \"displayName\": \"user1\", \"email\": \"User1@foo.com\", \"title\": \"user1\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "corpUserInfo", - "aspect": { - "value": "{\"active\": true, \"displayName\": \"User1\", \"email\": \"User1@foo.com\", \"title\": \"User1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "value": "{\"username\": \"User1@foo.com\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "corpUserKey", - "aspect": { - "value": "{\"username\": \"User1@foo.com\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "value": "{\"active\": true, \"displayName\": \"user2\", \"email\": \"User2@foo.com\", \"title\": \"user2\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "corpUserInfo", - "aspect": { - "value": "{\"active\": true, \"displayName\": \"User2\", \"email\": \"User2@foo.com\", \"title\": \"User2\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "value": "{\"username\": \"User2@foo.com\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "corpUserKey", - "aspect": { - "value": "{\"username\": \"User2@foo.com\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "value": "{\"customProperties\": {\"datasetId\": \"05169CD2-E713-41E6-9600-1D8066D95445\", \"reportId\": \"\", \"datasetWebUrl\": \"http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details\", \"createdFrom\": \"Dataset\"}, \"title\": \"test_tile\", \"description\": \"test_tile\", \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"inputs\": [{\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "value": "{\"customProperties\": {\"datasetId\": \"05169CD2-E713-41E6-9600-1D8066D95445\", \"reportId\": \"\", \"datasetWebUrl\": \"http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445\", \"createdFrom\": \"Dataset\"}, \"title\": \"test_tiles\", \"description\": \"test_tiles\", \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"inputs\": [{\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_book,DEV)\"}, {\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.member,DEV)\"}, {\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"chartId\": \"powerbi.linkedin.com/charts/B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "chartKey", - "aspect": { - "value": "{\"dashboardTool\": \"powerbi\", \"chartId\": \"powerbi.linkedin.com/charts/B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "value": "{\"paths\": [\"/powerbi/64ED5CAD-7C10-4684-8180-826122881108\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "browsePaths", - "aspect": { - "value": "{\"paths\": [\"/powerbi/64ED5CAD-7C10-4684-8180-826122881108\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"demo-workspace\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"foo\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dashboardKey", - "aspect": { - "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:users.User1@foo.com\", \"type\": \"NONE\"}, {\"owner\": \"urn:li:corpuser:users.User2@foo.com\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:users.User1@foo.com\", \"type\": \"CONSUMER\"}, {\"owner\": \"urn:li:corpuser:users.User2@foo.com\", \"type\": \"CONSUMER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "registryName": null, + "registryVersion": null, + "properties": null } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 4f2cde4cbf3d83..bbd60f856bd964 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -3,129 +3,238 @@ from freezegun import freeze_time from datahub.ingestion.run.pipeline import Pipeline -from datahub.ingestion.source.powerbi import PowerBiAPI from tests.test_helpers import mce_helpers FROZEN_TIME = "2022-02-03 07:00:00" -POWERBI_API = "datahub.ingestion.source.powerbi.PowerBiAPI" +def mock_msal_cca(*args, **kwargs): + class MsalClient: + def acquire_token_for_client(self, *args, **kwargs): + return { + "access_token": "dummy", + } -@freeze_time(FROZEN_TIME) -def test_powerbi_ingest(pytestconfig, tmp_path, mock_time): - mocked_client = mock.MagicMock() - # Set the datasource mapping dictionary - with mock.patch(POWERBI_API) as mock_sdk: - mock_sdk.return_value = mocked_client - # Mock the PowerBi Dashboard API response - mocked_client.get_workspace.return_value = PowerBiAPI.Workspace( - id="64ED5CAD-7C10-4684-8180-826122881108", - name="demo-workspace", - state="Active", - datasets={}, - dashboards=[ - PowerBiAPI.Dashboard( - id="7D668CAD-7FFC-4505-9215-655BCA5BEBAE", - displayName="test_dashboard", - isReadOnly=True, - embedUrl="https://localhost/dashboards/embed/1", - webUrl="https://localhost/dashboards/web/1", - workspace_id="4A378B07-FAA2-4EA2-9383-CBA91AD9681C", - workspace_name="foo", - users=[], - tiles=[ - PowerBiAPI.Tile( - id="B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0", - title="test_tiles", - embedUrl="https://localhost/tiles/embed/1", - createdFrom=PowerBiAPI.Tile.CreatedFrom.DATASET, - report=None, - dataset=PowerBiAPI.Dataset( - id="05169CD2-E713-41E6-9600-1D8066D95445", - name="library-dataset", - workspace_id="64ED5CAD-7C10-4684-8180-826122881108", - webUrl="http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445", - tables=[ - PowerBiAPI.Dataset.Table( - name="issue_book", schema_name="public" - ), - PowerBiAPI.Dataset.Table( - name="member", schema_name="public" - ), - PowerBiAPI.Dataset.Table( - name="issue_history", schema_name="public" - ), - ], - datasource=PowerBiAPI.DataSource( - id="DCE90B40-84D6-467A-9A5C-648E830E72D3", - database="library_db", - type="PostgreSql", - server="gs-library.postgres.database.azure.com", - metadata=PowerBiAPI.DataSource.MetaData( - is_relational=True - ), - ), - ), - ) - ], - ) - ], - ) + return MsalClient() - # Mock the PowerBi User API response - mocked_client.get_dashboard_users.return_value = [ - PowerBiAPI.User( - id="User1@foo.com", - displayName="User1", - emailAddress="User1@foo.com", - dashboardUserAccessRight="ReadWrite", - principalType="User", - graphId="C9EE53F2-88EA-4711-A173-AF0515A3CD46", - ), - PowerBiAPI.User( - id="User2@foo.com", - displayName="User2", - emailAddress="User2@foo.com", - dashboardUserAccessRight="ReadWrite", - principalType="User", - graphId="5ED26AA7-FCD2-42C5-BCE8-51E0AAD0682B", - ), - ] - - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" - - pipeline = Pipeline.create( - { - "run_id": "powerbi-test", - "source": { - "type": "powerbi", - "config": { - "client_id": "foo", - "client_secret": "bar", - "tenant_id": "0B0C960B-FCDF-4D0F-8C45-2E03BB59DDEB", - "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108", - "dataset_type_mapping": { - "PostgreSql": "postgres", - "Oracle": "oracle", + +def register_mock_api(request_mock): + api_vs_response = { + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": True, + "displayName": "test_dashboard", + "embedUrl": "https://localhost/dashboards/embed/1", + "webUrl": "https://localhost/dashboards/web/1", + } + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "identifier": "User1@foo.com", + "displayName": "user1", + "emailAddress": "User1@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", + "principalType": "User", + }, + { + "identifier": "User2@foo.com", + "displayName": "user2", + "emailAddress": "User2@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", + "principalType": "User", + }, + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0", + "title": "test_tile", + "embedUrl": "https://localhost/tiles/embed/1", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + }, + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": { + "method": "GET", + "status_code": 200, + "json": { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "name": "library-dataset", + "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445", + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3", + "datasourceType": "PostgreSql", + "connectionDetails": { + "database": "library_db", + "server": "foo", }, - "env": "DEV", }, - }, - "sink": { - "type": "file", - "config": { - "filename": f"{tmp_path}/powerbi_mces.json", + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED", + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "64ED5CAD-7C10-4684-8180-826122881108", + "name": "demo-workspace", + "state": "Active", + "datasets": [ + { + "id": "05169CD2-E713-41E6-9600-1D8066D95445", + "tables": [ + { + "name": "public issue_history", + "source": [ + { + "expression": "dummy", + } + ], + } + ], + } + ], }, - }, - } + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": { + "method": "POST", + "status_code": 200, + "json": { + "id": "4674efd1-603c-4129-8d82-03cf2be05aff", + }, + }, + } + + for url in api_vs_response.keys(): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], ) - pipeline.run() - pipeline.raise_from_status() - mce_out_file = "golden_test_ingest.json" - mce_helpers.check_golden_file( - pytestconfig, - output_path=tmp_path / "powerbi_mces.json", - golden_path=f"{test_resources_dir}/{mce_out_file}", - ) +def default_source_config(): + return { + "client_id": "foo", + "client_secret": "bar", + "tenant_id": "0B0C960B-FCDF-4D0F-8C45-2E03BB59DDEB", + "workspace_id": "64ED5CAD-7C10-4684-8180-826122881108", + "dataset_type_mapping": { + "PostgreSql": "postgres", + "Oracle": "oracle", + }, + "env": "DEV", + } + + +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +def test_powerbi_ingest(mock_msal, pytestconfig, tmp_path, mock_time, requests_mock): + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api(request_mock=requests_mock) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + mce_out_file = "golden_test_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "powerbi_mces.json", + golden_path=f"{test_resources_dir}/{mce_out_file}", + ) + + +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +def test_override_ownership( + mock_msal, pytestconfig, tmp_path, mock_time, requests_mock +): + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api(request_mock=requests_mock) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + "extract_ownership": False, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_mces_disabled_ownership.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + mce_out_file = "golden_test_disabled_ownership.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "powerbi_mces_disabled_ownership.json", + golden_path=f"{test_resources_dir}/{mce_out_file}", + ) diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index 0c639da5991575..905bf4f7ca010f 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -52,7 +52,7 @@ def loaded_presto_on_hive(presto_on_hive_runner): @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_presto_on_hive_ingest( loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time ): @@ -112,7 +112,7 @@ def test_presto_on_hive_ingest( @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_presto_on_hive_instance_ingest( loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time ): diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json index fec72a5725cde8..dd15558415b694 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json @@ -49,6 +49,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -59,6 +61,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -66,6 +69,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -76,6 +81,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -83,6 +89,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -93,6 +101,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -100,6 +109,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -110,6 +121,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -117,6 +129,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -127,6 +141,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -763,7 +778,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", "contentType": "application/json" }, "systemMetadata": { @@ -824,6 +839,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -834,6 +851,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -841,6 +859,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -851,6 +871,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -858,6 +879,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -868,6 +891,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -875,6 +899,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -885,6 +911,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -892,6 +919,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -902,6 +931,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -949,7 +979,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06428571428571428, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.12857142857142856, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json index 1307c2559e7fe9..88c640d4e39c41 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json @@ -49,6 +49,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -59,6 +61,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -66,6 +69,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -76,6 +81,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -83,6 +89,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -93,6 +101,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -100,6 +109,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -110,6 +121,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -117,6 +129,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -127,6 +141,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -763,7 +778,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json index 3e3a470d30f974..94af91a2ebcfaf 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json @@ -49,6 +49,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -59,6 +61,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -66,6 +69,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -76,6 +81,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -83,6 +89,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -93,6 +101,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -100,6 +109,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -110,6 +121,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -117,6 +129,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -127,6 +141,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -763,7 +778,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", "contentType": "application/json" }, "systemMetadata": { @@ -824,6 +839,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -834,6 +851,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -841,6 +859,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -851,6 +871,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -858,6 +879,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -868,6 +891,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -875,6 +899,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -885,6 +911,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -892,6 +919,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -902,6 +931,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -949,7 +979,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06428571428571428, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.12857142857142856, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json index c82ca615ece3b2..a804625712cffd 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json @@ -49,6 +49,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -59,6 +61,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -66,6 +69,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -76,6 +81,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -83,6 +89,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -93,6 +101,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -100,6 +109,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -110,6 +121,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -117,6 +129,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -127,6 +141,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -763,7 +778,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json index 6e86c1235a2023..96cb319e812cb2 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json @@ -49,6 +49,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -61,6 +63,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -68,6 +71,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -80,6 +85,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -87,6 +93,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -97,6 +105,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -104,6 +113,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -114,6 +125,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -121,6 +133,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -131,6 +145,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -138,6 +153,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -148,6 +165,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -155,6 +173,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -165,6 +185,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -172,6 +193,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -182,6 +205,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -189,6 +213,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -199,6 +225,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -206,6 +233,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -218,6 +247,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -225,6 +255,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -235,6 +267,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -242,6 +275,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -252,6 +287,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -259,6 +295,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -269,6 +307,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -276,6 +315,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -286,6 +327,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -293,6 +335,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -303,6 +347,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -310,6 +355,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -322,6 +369,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -329,6 +377,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -339,6 +389,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -346,6 +397,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -356,6 +409,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -363,6 +417,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -373,6 +429,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -380,6 +437,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -390,6 +449,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -397,6 +457,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -407,6 +469,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -414,6 +477,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -424,6 +489,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -431,6 +497,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -441,6 +509,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -448,6 +517,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -458,6 +529,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -465,6 +537,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -475,6 +549,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -482,6 +557,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -492,6 +569,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -499,6 +577,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -509,6 +589,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -516,6 +597,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -526,6 +609,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -533,6 +617,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -543,6 +629,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -550,6 +637,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -562,6 +651,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -569,6 +659,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -579,6 +671,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -586,6 +679,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -596,6 +691,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -603,6 +699,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -613,6 +711,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -620,6 +719,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -630,6 +731,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -637,6 +739,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -649,6 +753,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -656,6 +761,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -666,6 +773,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -673,6 +781,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -683,6 +793,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -690,6 +801,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -700,6 +813,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -707,6 +821,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -717,6 +833,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -724,6 +841,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -734,6 +853,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1370,7 +1490,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 12, \"columnCount\": 11, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 12}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 12}], \"sampleValues\": [\"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}, {\"fieldPath\": \"year\", \"uniqueCount\": 3, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"2019\", \"frequency\": 4}, {\"value\": \"2020\", \"frequency\": 4}, {\"value\": \"2021\", \"frequency\": 4}], \"sampleValues\": [\"2019\", \"2019\", \"2019\", \"2019\", \"2020\", \"2020\", \"2020\", \"2020\", \"2021\", \"2021\", \"2021\", \"2021\"]}, {\"fieldPath\": \"month\", \"uniqueCount\": 4, \"uniqueProportion\": 0.3333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"april\", \"frequency\": 2}, {\"value\": \"feb\", \"frequency\": 4}, {\"value\": \"jan\", \"frequency\": 2}, {\"value\": \"march\", \"frequency\": 4}], \"sampleValues\": [\"april\", \"april\", \"feb\", \"feb\", \"feb\", \"feb\", \"jan\", \"jan\", \"march\", \"march\", \"march\", \"march\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 1, \"columnCount\": 9, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}], \"sampleValues\": [\"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 1}], \"sampleValues\": [\"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json index e40ddd345312b6..3565f611a047fe 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json @@ -49,6 +49,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -61,6 +63,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -68,6 +71,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -80,6 +85,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -87,6 +93,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -97,6 +105,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -104,6 +113,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -114,6 +125,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -121,6 +133,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -131,6 +145,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -138,6 +153,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -148,6 +165,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -155,6 +173,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -165,6 +185,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -172,6 +193,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -182,6 +205,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -189,6 +213,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -199,6 +225,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -206,6 +233,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -218,6 +247,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -225,6 +255,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -235,6 +267,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -242,6 +275,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -252,6 +287,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -259,6 +295,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -269,6 +307,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -276,6 +315,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -286,6 +327,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -293,6 +335,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -303,6 +347,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -310,6 +355,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -322,6 +369,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -329,6 +377,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -339,6 +389,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -346,6 +397,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -356,6 +409,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -363,6 +417,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -373,6 +429,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -380,6 +437,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -390,6 +449,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -397,6 +457,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -407,6 +469,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -414,6 +477,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -424,6 +489,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -431,6 +497,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -441,6 +509,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -448,6 +517,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -458,6 +529,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -465,6 +537,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -475,6 +549,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -482,6 +557,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -492,6 +569,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -499,6 +577,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -509,6 +589,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -516,6 +597,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -526,6 +609,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -533,6 +617,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -543,6 +629,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -550,6 +637,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -562,6 +651,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -569,6 +659,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -579,6 +671,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -586,6 +679,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -596,6 +691,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -603,6 +699,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -613,6 +711,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -620,6 +719,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -630,6 +731,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -637,6 +739,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.ArrayType": { @@ -649,6 +753,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -656,6 +761,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -666,6 +773,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -673,6 +781,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.RecordType": {} @@ -683,6 +793,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -690,6 +801,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -700,6 +813,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -707,6 +821,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -717,6 +833,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -724,6 +841,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -734,6 +853,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1370,7 +1490,7 @@ "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 12, \"columnCount\": 11, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 12}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 12}], \"sampleValues\": [\"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}, {\"fieldPath\": \"year\", \"uniqueCount\": 3, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"2019\", \"frequency\": 4}, {\"value\": \"2020\", \"frequency\": 4}, {\"value\": \"2021\", \"frequency\": 4}], \"sampleValues\": [\"2019\", \"2019\", \"2019\", \"2019\", \"2020\", \"2020\", \"2020\", \"2020\", \"2021\", \"2021\", \"2021\", \"2021\"]}, {\"fieldPath\": \"month\", \"uniqueCount\": 4, \"uniqueProportion\": 0.3333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"april\", \"frequency\": 2}, {\"value\": \"feb\", \"frequency\": 4}, {\"value\": \"jan\", \"frequency\": 2}, {\"value\": \"march\", \"frequency\": 4}], \"sampleValues\": [\"april\", \"april\", \"feb\", \"feb\", \"feb\", \"feb\", \"jan\", \"jan\", \"march\", \"march\", \"march\", \"march\"]}]}", + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 1, \"columnCount\": 9, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}], \"sampleValues\": [\"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 1}], \"sampleValues\": [\"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}]}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py index 6076daca1cb824..d4e840c7e837e6 100644 --- a/metadata-ingestion/tests/integration/s3/test_s3.py +++ b/metadata-ingestion/tests/integration/s3/test_s3.py @@ -3,7 +3,7 @@ import os import pytest -from boto3 import Session +from boto3.session import Session from moto import mock_s3 from pydantic import ValidationError @@ -69,6 +69,7 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_name): source_files = os.listdir(SOURCE_FILES_PATH) +@pytest.mark.slow_unit @pytest.mark.parametrize("source_file", source_files) def test_data_lake_s3_ingest( pytestconfig, s3_populate, source_file, tmp_path, mock_time @@ -101,6 +102,7 @@ def test_data_lake_s3_ingest( ) +@pytest.mark.slow_unit @pytest.mark.parametrize("source_file", source_files) def test_data_lake_local_ingest(pytestconfig, source_file, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/s3/" diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/account_custom_fields_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/account_custom_fields_soql_response.json new file mode 100644 index 00000000000000..50142ca7edec15 --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/account_custom_fields_soql_response.json @@ -0,0 +1,177 @@ +{ + "size": 7, + "totalSize": 7, + "done": true, + "queryLocator": null, + "entityTypeName": "CustomField", + "records": [ + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8LEAW" + }, + "DeveloperName": "CustomerPriority", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8NEAW" + }, + "DeveloperName": "SLA", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8KEAW" + }, + "DeveloperName": "Active", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8MEAW" + }, + "DeveloperName": "NumberofLocations", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8QEAW" + }, + "DeveloperName": "UpsellOpportunity", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8PEAW" + }, + "DeveloperName": "SLASerialNumber", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004FO8OEAW" + }, + "DeveloperName": "SLAExpirationDate", + "CreatedDate": "2022-05-02T03:22:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json new file mode 100644 index 00000000000000..947761b8c79a0f --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json @@ -0,0 +1,2513 @@ +{ + "size": 72, + "totalSize": 72, + "done": true, + "queryLocator": null, + "entityTypeName": "EntityParticle", + "records": [ + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Id" + }, + "QualifiedApiName": "Id", + "DeveloperName": "Id", + "Label": "Account ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Id" + }, + "DataType": "Lookup()", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "id", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.IsDeleted" + }, + "QualifiedApiName": "IsDeleted", + "DeveloperName": "IsDeleted", + "Label": "Deleted", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.IsDeleted" + }, + "DataType": "Checkbox", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "boolean", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.MasterRecord" + }, + "QualifiedApiName": "MasterRecordId", + "DeveloperName": "MasterRecord", + "Label": "Master Record ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.MasterRecord" + }, + "DataType": "Lookup(Account)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "Account" + ] + }, + "RelationshipName": "MasterRecord", + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Name.TextName" + }, + "QualifiedApiName": "Name", + "DeveloperName": "Name", + "Label": "Account Name", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Name" + }, + "DataType": "Name", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.CompareName" + }, + "QualifiedApiName": "CompareName", + "DeveloperName": "CompareName", + "Label": "Compare Name", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.CompareName" + }, + "DataType": "Text(80)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Type" + }, + "QualifiedApiName": "Type", + "DeveloperName": "Type", + "Label": "Account Type", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Type" + }, + "DataType": "Picklist", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Parent" + }, + "QualifiedApiName": "ParentId", + "DeveloperName": "Parent", + "Label": "Parent Account ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Parent" + }, + "DataType": "Hierarchy", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "Account" + ] + }, + "RelationshipName": "Parent", + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingStreet" + }, + "QualifiedApiName": "BillingStreet", + "DeveloperName": "BillingAddress", + "Label": "Billing Street", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "textarea", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingCity" + }, + "QualifiedApiName": "BillingCity", + "DeveloperName": "BillingAddress", + "Label": "Billing City", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingState" + }, + "QualifiedApiName": "BillingState", + "DeveloperName": "BillingAddress", + "Label": "Billing State/Province", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingPostalCode" + }, + "QualifiedApiName": "BillingPostalCode", + "DeveloperName": "BillingAddress", + "Label": "Billing Zip/Postal Code", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 20, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingCountry" + }, + "QualifiedApiName": "BillingCountry", + "DeveloperName": "BillingAddress", + "Label": "Billing Country", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingLatitude" + }, + "QualifiedApiName": "BillingLatitude", + "DeveloperName": "BillingAddress", + "Label": "Billing Latitude", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "double", + "Precision": 18, + "Scale": 15, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingLongitude" + }, + "QualifiedApiName": "BillingLongitude", + "DeveloperName": "BillingAddress", + "Label": "Billing Longitude", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "double", + "Precision": 18, + "Scale": 15, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress.BillingGeocodeAccuracy" + }, + "QualifiedApiName": "BillingGeocodeAccuracy", + "DeveloperName": "BillingAddress", + "Label": "Billing Geocode Accuracy", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.BillingAddress" + }, + "QualifiedApiName": "BillingAddress", + "DeveloperName": "BillingAddress", + "Label": "Billing Address", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.BillingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "address", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": true, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingStreet" + }, + "QualifiedApiName": "ShippingStreet", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Street", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "textarea", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingCity" + }, + "QualifiedApiName": "ShippingCity", + "DeveloperName": "ShippingAddress", + "Label": "Shipping City", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingState" + }, + "QualifiedApiName": "ShippingState", + "DeveloperName": "ShippingAddress", + "Label": "Shipping State/Province", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingPostalCode" + }, + "QualifiedApiName": "ShippingPostalCode", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Zip/Postal Code", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 20, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingCountry" + }, + "QualifiedApiName": "ShippingCountry", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Country", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingLatitude" + }, + "QualifiedApiName": "ShippingLatitude", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Latitude", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "double", + "Precision": 18, + "Scale": 15, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingLongitude" + }, + "QualifiedApiName": "ShippingLongitude", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Longitude", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "double", + "Precision": 18, + "Scale": 15, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress.ShippingGeocodeAccuracy" + }, + "QualifiedApiName": "ShippingGeocodeAccuracy", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Geocode Accuracy", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": true, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ShippingAddress" + }, + "QualifiedApiName": "ShippingAddress", + "DeveloperName": "ShippingAddress", + "Label": "Shipping Address", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ShippingAddress" + }, + "DataType": "Address", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "address", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": true, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Phone" + }, + "QualifiedApiName": "Phone", + "DeveloperName": "Phone", + "Label": "Account Phone", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Phone" + }, + "DataType": "Phone", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "phone", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Fax" + }, + "QualifiedApiName": "Fax", + "DeveloperName": "Fax", + "Label": "Account Fax", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Fax" + }, + "DataType": "Fax", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "phone", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.AccountNumber" + }, + "QualifiedApiName": "AccountNumber", + "DeveloperName": "AccountNumber", + "Label": "Account Number", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.AccountNumber" + }, + "DataType": "Text(40)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Website" + }, + "QualifiedApiName": "Website", + "DeveloperName": "Website", + "Label": "Website", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Website" + }, + "DataType": "URL(255)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "url", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.PhotoUrl" + }, + "QualifiedApiName": "PhotoUrl", + "DeveloperName": "PhotoUrl", + "Label": "Photo URL", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.PhotoUrl" + }, + "DataType": "URL(255)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "url", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Sic" + }, + "QualifiedApiName": "Sic", + "DeveloperName": "Sic", + "Label": "SIC Code", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Sic" + }, + "DataType": "Text(20)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 20, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Industry" + }, + "QualifiedApiName": "Industry", + "DeveloperName": "Industry", + "Label": "Industry", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Industry" + }, + "DataType": "Picklist", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.AnnualRevenue" + }, + "QualifiedApiName": "AnnualRevenue", + "DeveloperName": "AnnualRevenue", + "Label": "Annual Revenue", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.AnnualRevenue" + }, + "DataType": "Currency(18, 0)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "currency", + "Precision": 18, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.NumberOfEmployees" + }, + "QualifiedApiName": "NumberOfEmployees", + "DeveloperName": "NumberOfEmployees", + "Label": "Employees", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.NumberOfEmployees" + }, + "DataType": "Number(8, 0)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "integer", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 8, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Ownership" + }, + "QualifiedApiName": "Ownership", + "DeveloperName": "Ownership", + "Label": "Ownership", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Ownership" + }, + "DataType": "Picklist", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.TickerSymbol" + }, + "QualifiedApiName": "TickerSymbol", + "DeveloperName": "TickerSymbol", + "Label": "Ticker Symbol", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.TickerSymbol" + }, + "DataType": "Content(20)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 20, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Description" + }, + "QualifiedApiName": "Description", + "DeveloperName": "Description", + "Label": "Account Description", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Description" + }, + "DataType": "Long Text Area(32000)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "textarea", + "Precision": 0, + "Scale": 0, + "Length": 32000, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Rating" + }, + "QualifiedApiName": "Rating", + "DeveloperName": "Rating", + "Label": "Account Rating", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Rating" + }, + "DataType": "Picklist", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Site" + }, + "QualifiedApiName": "Site", + "DeveloperName": "Site", + "Label": "Account Site", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Site" + }, + "DataType": "Text(80)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.CompareSite" + }, + "QualifiedApiName": "CompareSite", + "DeveloperName": "CompareSite", + "Label": "Compare Site", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.CompareSite" + }, + "DataType": "Text(80)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Owner" + }, + "QualifiedApiName": "OwnerId", + "DeveloperName": "Owner", + "Label": "Owner ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Owner" + }, + "DataType": "Lookup(User)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "User" + ] + }, + "RelationshipName": "Owner", + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.OwnerAlias" + }, + "QualifiedApiName": "OwnerAlias", + "DeveloperName": "OwnerAlias", + "Label": "Owner Alias", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.OwnerAlias" + }, + "DataType": "Text(30)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 30, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.CreatedDate" + }, + "QualifiedApiName": "CreatedDate", + "DeveloperName": "CreatedDate", + "Label": "Created Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.CreatedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.CreatedBy" + }, + "QualifiedApiName": "CreatedById", + "DeveloperName": "CreatedBy", + "Label": "Created By ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.CreatedBy" + }, + "DataType": "Lookup(User)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "User" + ] + }, + "RelationshipName": "CreatedBy", + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.LastModifiedDate" + }, + "QualifiedApiName": "LastModifiedDate", + "DeveloperName": "LastModifiedDate", + "Label": "Last Modified Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.LastModifiedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.LastModifiedBy" + }, + "QualifiedApiName": "LastModifiedById", + "DeveloperName": "LastModifiedBy", + "Label": "Last Modified By ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.LastModifiedBy" + }, + "DataType": "Lookup(User)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "User" + ] + }, + "RelationshipName": "LastModifiedBy", + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.SystemModstamp" + }, + "QualifiedApiName": "SystemModstamp", + "DeveloperName": "SystemModstamp", + "Label": "System Modstamp", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.SystemModstamp" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.LastActivityDate" + }, + "QualifiedApiName": "LastActivityDate", + "DeveloperName": "LastActivityDate", + "Label": "Last Activity", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.LastActivityDate" + }, + "DataType": "Date", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "date", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.LastViewedDate" + }, + "QualifiedApiName": "LastViewedDate", + "DeveloperName": "LastViewedDate", + "Label": "Last Viewed Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.LastViewedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.LastReferencedDate" + }, + "QualifiedApiName": "LastReferencedDate", + "DeveloperName": "LastReferencedDate", + "Label": "Last Referenced Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.LastReferencedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Jigsaw" + }, + "QualifiedApiName": "Jigsaw", + "DeveloperName": "Jigsaw", + "Label": "Data.com Key", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Jigsaw" + }, + "DataType": "Text(20)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 20, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.JigsawCompany" + }, + "QualifiedApiName": "JigsawCompanyId", + "DeveloperName": "JigsawCompany", + "Label": "Jigsaw Company ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.JigsawCompany" + }, + "DataType": "External Lookup", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 20, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": "JigsawCompany", + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.CleanStatus" + }, + "QualifiedApiName": "CleanStatus", + "DeveloperName": "CleanStatus", + "Label": "Clean Status", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.CleanStatus" + }, + "DataType": "Picklist", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 40, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.AccountSource" + }, + "QualifiedApiName": "AccountSource", + "DeveloperName": "AccountSource", + "Label": "Account Source", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.AccountSource" + }, + "DataType": "Picklist", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.DunsNumber" + }, + "QualifiedApiName": "DunsNumber", + "DeveloperName": "DunsNumber", + "Label": "D-U-N-S Number", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.DunsNumber" + }, + "DataType": "Text(9)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 9, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Tradestyle" + }, + "QualifiedApiName": "Tradestyle", + "DeveloperName": "Tradestyle", + "Label": "Tradestyle", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Tradestyle" + }, + "DataType": "Text(255)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.NaicsCode" + }, + "QualifiedApiName": "NaicsCode", + "DeveloperName": "NaicsCode", + "Label": "NAICS Code", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.NaicsCode" + }, + "DataType": "Text(8)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 8, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.NaicsDesc" + }, + "QualifiedApiName": "NaicsDesc", + "DeveloperName": "NaicsDesc", + "Label": "NAICS Description", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.NaicsDesc" + }, + "DataType": "Text(120)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 120, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.YearStarted" + }, + "QualifiedApiName": "YearStarted", + "DeveloperName": "YearStarted", + "Label": "Year Started", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.YearStarted" + }, + "DataType": "Text(4)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 4, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.SicDesc" + }, + "QualifiedApiName": "SicDesc", + "DeveloperName": "SicDesc", + "Label": "SIC Description", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.SicDesc" + }, + "DataType": "Text(80)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.DandbCompany" + }, + "QualifiedApiName": "DandbCompanyId", + "DeveloperName": "DandbCompany", + "Label": "D&B Company ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.DandbCompany" + }, + "DataType": "Lookup(D&B Company)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "DandBCompany" + ] + }, + "RelationshipName": "DandbCompany", + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.OperatingHours" + }, + "QualifiedApiName": "OperatingHoursId", + "DeveloperName": "OperatingHours", + "Label": "Operating Hour ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.OperatingHours" + }, + "DataType": "Lookup(Operating Hours)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "OperatingHours" + ] + }, + "RelationshipName": "OperatingHours", + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ConnectionReceivedDate" + }, + "QualifiedApiName": "ConnectionReceivedDate", + "DeveloperName": "ConnectionReceivedDate", + "Label": "Received Connection Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ConnectionReceivedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.ConnectionSentDate" + }, + "QualifiedApiName": "ConnectionSentDate", + "DeveloperName": "ConnectionSentDate", + "Label": "Sent Connection Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.ConnectionSentDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.Tier" + }, + "QualifiedApiName": "Tier", + "DeveloperName": "Tier", + "Label": "Einstein Account Tier", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.Tier" + }, + "DataType": "Text(2)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 2, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8L" + }, + "QualifiedApiName": "CustomerPriority__c", + "DeveloperName": "CustomerPriority", + "Label": "Customer Priority", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8L" + }, + "DataType": "Picklist", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8N" + }, + "QualifiedApiName": "SLA__c", + "DeveloperName": "SLA", + "Label": "SLA", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8N" + }, + "DataType": "Picklist", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8K" + }, + "QualifiedApiName": "Active__c", + "DeveloperName": "Active", + "Label": "Active", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8K" + }, + "DataType": "Picklist", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8M" + }, + "QualifiedApiName": "NumberofLocations__c", + "DeveloperName": "NumberofLocations", + "Label": "Number of Locations", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8M" + }, + "DataType": "Number(3, 0)", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "double", + "Precision": 3, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8Q" + }, + "QualifiedApiName": "UpsellOpportunity__c", + "DeveloperName": "UpsellOpportunity", + "Label": "Upsell Opportunity", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8Q" + }, + "DataType": "Picklist", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8P" + }, + "QualifiedApiName": "SLASerialNumber__c", + "DeveloperName": "SLASerialNumber", + "Label": "SLA Serial Number", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8P" + }, + "DataType": "Text(10)", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 10, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/Account.00N5i000004FO8O" + }, + "QualifiedApiName": "SLAExpirationDate__c", + "DeveloperName": "SLAExpirationDate", + "Label": "SLA Expiration Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/Account.00N5i000004FO8O" + }, + "DataType": "Date", + "LastModifiedDate": "2022-05-02T03:22:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "date", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/entity_definition_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/entity_definition_soql_response.json new file mode 100644 index 00000000000000..efae809c1c78f2 --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/entity_definition_soql_response.json @@ -0,0 +1,2151 @@ +{ + "size": 153, + "totalSize": 153, + "done": true, + "queryLocator": null, + "entityTypeName": "EntityDefinition", + "records": [ + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Contract" + }, + "DurableId": "Contract", + "QualifiedApiName": "Contract", + "DeveloperName": "Contract", + "Label": "Contract", + "PluralLabel": "Contracts", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Order" + }, + "DurableId": "Order", + "QualifiedApiName": "Order", + "DeveloperName": "Order", + "Label": "Order", + "PluralLabel": "Orders", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/OrderItem" + }, + "DurableId": "OrderItem", + "QualifiedApiName": "OrderItem", + "DeveloperName": "OrderItem", + "Label": "Order Product", + "PluralLabel": "Order Products", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Campaign" + }, + "DurableId": "Campaign", + "QualifiedApiName": "Campaign", + "DeveloperName": "Campaign", + "Label": "Campaign", + "PluralLabel": "Campaigns", + "InternalSharingModel": "FullAccess", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CampaignMember" + }, + "DurableId": "CampaignMember", + "QualifiedApiName": "CampaignMember", + "DeveloperName": "CampaignMember", + "Label": "Campaign Member", + "PluralLabel": "Campaign Members", + "InternalSharingModel": "ControlledByCampaign", + "ExternalSharingModel": "ControlledByCampaign", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Account" + }, + "DurableId": "Account", + "QualifiedApiName": "Account", + "DeveloperName": "Account", + "Label": "Account", + "PluralLabel": "Accounts", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Contact" + }, + "DurableId": "Contact", + "QualifiedApiName": "Contact", + "DeveloperName": "Contact", + "Label": "Contact", + "PluralLabel": "Contacts", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Lead" + }, + "DurableId": "Lead", + "QualifiedApiName": "Lead", + "DeveloperName": "Lead", + "Label": "Lead", + "PluralLabel": "Leads", + "InternalSharingModel": "ReadWriteTransfer", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Opportunity" + }, + "DurableId": "Opportunity", + "QualifiedApiName": "Opportunity", + "DeveloperName": "Opportunity", + "Label": "Opportunity", + "PluralLabel": "Opportunities", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/OpportunityContactRole" + }, + "DurableId": "OpportunityContactRole", + "QualifiedApiName": "OpportunityContactRole", + "DeveloperName": "OpportunityContactRole", + "Label": "Opportunity Contact Role", + "PluralLabel": "Opportunity Contact Role", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/OpportunityLineItem" + }, + "DurableId": "OpportunityLineItem", + "QualifiedApiName": "OpportunityLineItem", + "DeveloperName": "OpportunityLineItem", + "Label": "Opportunity Product", + "PluralLabel": "Opportunity Product", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PricebookEntry" + }, + "DurableId": "PricebookEntry", + "QualifiedApiName": "PricebookEntry", + "DeveloperName": "PricebookEntry", + "Label": "Price Book Entry", + "PluralLabel": "Price Book Entries", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Product2" + }, + "DurableId": "Product2", + "QualifiedApiName": "Product2", + "DeveloperName": "Product2", + "Label": "Product", + "PluralLabel": "Products", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Asset" + }, + "DurableId": "Asset", + "QualifiedApiName": "Asset", + "DeveloperName": "Asset", + "Label": "Asset", + "PluralLabel": "Assets", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Case" + }, + "DurableId": "Case", + "QualifiedApiName": "Case", + "DeveloperName": "Case", + "Label": "Case", + "PluralLabel": "Cases", + "InternalSharingModel": "ReadWriteTransfer", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Solution" + }, + "DurableId": "Solution", + "QualifiedApiName": "Solution", + "DeveloperName": "Solution", + "Label": "Solution", + "PluralLabel": "Solutions", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContentVersion" + }, + "DurableId": "ContentVersion", + "QualifiedApiName": "ContentVersion", + "DeveloperName": "ContentVersion", + "Label": "Content Version", + "PluralLabel": "Content Versions", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Task" + }, + "DurableId": "Task", + "QualifiedApiName": "Task", + "DeveloperName": "Task", + "Label": "Task", + "PluralLabel": "Tasks", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Event" + }, + "DurableId": "Event", + "QualifiedApiName": "Event", + "DeveloperName": "Event", + "Label": "Event", + "PluralLabel": "Events", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/User" + }, + "DurableId": "User", + "QualifiedApiName": "User", + "DeveloperName": "User", + "Label": "User", + "PluralLabel": "Users", + "InternalSharingModel": "Read", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/EmailMessage" + }, + "DurableId": "EmailMessage", + "QualifiedApiName": "EmailMessage", + "DeveloperName": "EmailMessage", + "Label": "Email Message", + "PluralLabel": "Email Messages", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/VoiceCall" + }, + "DurableId": "VoiceCall", + "QualifiedApiName": "VoiceCall", + "DeveloperName": "VoiceCall", + "Label": "Voice Call", + "PluralLabel": "Voice Calls", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Individual" + }, + "DurableId": "Individual", + "QualifiedApiName": "Individual", + "DeveloperName": "Individual", + "Label": "Individual", + "PluralLabel": "Individuals", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Invoice" + }, + "DurableId": "Invoice", + "QualifiedApiName": "Invoice", + "DeveloperName": "Invoice", + "Label": "Invoice", + "PluralLabel": "Invoices", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Payment" + }, + "DurableId": "Payment", + "QualifiedApiName": "Payment", + "DeveloperName": "Payment", + "Label": "Payment", + "PluralLabel": "Payments", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Pricebook2" + }, + "DurableId": "Pricebook2", + "QualifiedApiName": "Pricebook2", + "DeveloperName": "Pricebook2", + "Label": "Price Book", + "PluralLabel": "Price Books", + "InternalSharingModel": "ReadSelect", + "ExternalSharingModel": "ReadSelect", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Idea" + }, + "DurableId": "Idea", + "QualifiedApiName": "Idea", + "DeveloperName": "Idea", + "Label": "Idea", + "PluralLabel": "Ideas", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Macro" + }, + "DurableId": "Macro", + "QualifiedApiName": "Macro", + "DeveloperName": "Macro", + "Label": "Macro", + "PluralLabel": "Macros", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkOrder" + }, + "DurableId": "WorkOrder", + "QualifiedApiName": "WorkOrder", + "DeveloperName": "WorkOrder", + "Label": "Work Order", + "PluralLabel": "Work Orders", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkOrderLineItem" + }, + "DurableId": "WorkOrderLineItem", + "QualifiedApiName": "WorkOrderLineItem", + "DeveloperName": "WorkOrderLineItem", + "Label": "Work Order Line Item", + "PluralLabel": "Work Order Line Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceAppointment" + }, + "DurableId": "ServiceAppointment", + "QualifiedApiName": "ServiceAppointment", + "DeveloperName": "ServiceAppointment", + "Label": "Service Appointment", + "PluralLabel": "Service Appointments", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkType" + }, + "DurableId": "WorkType", + "QualifiedApiName": "WorkType", + "DeveloperName": "WorkType", + "Label": "Work Type", + "PluralLabel": "Work Types", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceResource" + }, + "DurableId": "ServiceResource", + "QualifiedApiName": "ServiceResource", + "DeveloperName": "ServiceResource", + "Label": "Service Resource", + "PluralLabel": "Service Resources", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceTerritory" + }, + "DurableId": "ServiceTerritory", + "QualifiedApiName": "ServiceTerritory", + "DeveloperName": "ServiceTerritory", + "Label": "Service Territory", + "PluralLabel": "Service Territories", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceTerritoryMember" + }, + "DurableId": "ServiceTerritoryMember", + "QualifiedApiName": "ServiceTerritoryMember", + "DeveloperName": "ServiceTerritoryMember", + "Label": "Service Territory Member", + "PluralLabel": "Service Territory Members", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceResourceSkill" + }, + "DurableId": "ServiceResourceSkill", + "QualifiedApiName": "ServiceResourceSkill", + "DeveloperName": "ServiceResourceSkill", + "Label": "Service Resource Skill", + "PluralLabel": "Service Resource Skills", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/SkillRequirement" + }, + "DurableId": "SkillRequirement", + "QualifiedApiName": "SkillRequirement", + "DeveloperName": "SkillRequirement", + "Label": "Skill Requirement", + "PluralLabel": "Skill Requirements", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AssignedResource" + }, + "DurableId": "AssignedResource", + "QualifiedApiName": "AssignedResource", + "DeveloperName": "AssignedResource", + "Label": "Assigned Resource", + "PluralLabel": "Assigned Resources", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/OperatingHours" + }, + "DurableId": "OperatingHours", + "QualifiedApiName": "OperatingHours", + "DeveloperName": "OperatingHours", + "Label": "Operating Hours", + "PluralLabel": "Operating Hours", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ResourceAbsence" + }, + "DurableId": "ResourceAbsence", + "QualifiedApiName": "ResourceAbsence", + "DeveloperName": "ResourceAbsence", + "Label": "Resource Absence", + "PluralLabel": "Resource Absences", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/TimeSlot" + }, + "DurableId": "TimeSlot", + "QualifiedApiName": "TimeSlot", + "DeveloperName": "TimeSlot", + "Label": "Time Slot", + "PluralLabel": "Time Slots", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ResourcePreference" + }, + "DurableId": "ResourcePreference", + "QualifiedApiName": "ResourcePreference", + "DeveloperName": "ResourcePreference", + "Label": "Resource Preference", + "PluralLabel": "Resource Preferences", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Entitlement" + }, + "DurableId": "Entitlement", + "QualifiedApiName": "Entitlement", + "DeveloperName": "Entitlement", + "Label": "Entitlement", + "PluralLabel": "Entitlements", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/EntityMilestone" + }, + "DurableId": "EntityMilestone", + "QualifiedApiName": "EntityMilestone", + "DeveloperName": "EntityMilestone", + "Label": "Object Milestone", + "PluralLabel": "Object Milestones", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceContract" + }, + "DurableId": "ServiceContract", + "QualifiedApiName": "ServiceContract", + "DeveloperName": "ServiceContract", + "Label": "Service Contract", + "PluralLabel": "Service Contracts", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContractLineItem" + }, + "DurableId": "ContractLineItem", + "QualifiedApiName": "ContractLineItem", + "DeveloperName": "ContractLineItem", + "Label": "Contract Line Item", + "PluralLabel": "Contract Line Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Location" + }, + "DurableId": "Location", + "QualifiedApiName": "Location", + "DeveloperName": "Location", + "Label": "Location", + "PluralLabel": "Locations", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AssociatedLocation" + }, + "DurableId": "AssociatedLocation", + "QualifiedApiName": "AssociatedLocation", + "DeveloperName": "AssociatedLocation", + "Label": "Associated Location", + "PluralLabel": "Associated Locations", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/DandBCompany" + }, + "DurableId": "DandBCompany", + "QualifiedApiName": "DandBCompany", + "DeveloperName": "DandBCompany", + "Label": "D&B Company", + "PluralLabel": "D&B Companies", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AccountCleanInfo" + }, + "DurableId": "AccountCleanInfo", + "QualifiedApiName": "AccountCleanInfo", + "DeveloperName": "AccountCleanInfo", + "Label": "Account Clean Info", + "PluralLabel": "Account Clean Info", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactCleanInfo" + }, + "DurableId": "ContactCleanInfo", + "QualifiedApiName": "ContactCleanInfo", + "DeveloperName": "ContactCleanInfo", + "Label": "Contact Clean Info", + "PluralLabel": "Contact Clean Info", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LeadCleanInfo" + }, + "DurableId": "LeadCleanInfo", + "QualifiedApiName": "LeadCleanInfo", + "DeveloperName": "LeadCleanInfo", + "Label": "Lead Clean Info", + "PluralLabel": "Lead Clean Info", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/QuickText" + }, + "DurableId": "QuickText", + "QualifiedApiName": "QuickText", + "DeveloperName": "QuickText", + "Label": "Quick Text", + "PluralLabel": "Quick Text", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/SocialPersona" + }, + "DurableId": "SocialPersona", + "QualifiedApiName": "SocialPersona", + "DeveloperName": "SocialPersona", + "Label": "Social Persona", + "PluralLabel": "Social Personas", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/UserProvisioningRequest" + }, + "DurableId": "UserProvisioningRequest", + "QualifiedApiName": "UserProvisioningRequest", + "DeveloperName": "UserProvisioningRequest", + "Label": "User Provisioning Request", + "PluralLabel": "User Provisioning Requests", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/DuplicateRecordSet" + }, + "DurableId": "DuplicateRecordSet", + "QualifiedApiName": "DuplicateRecordSet", + "DeveloperName": "DuplicateRecordSet", + "Label": "Duplicate Record Set", + "PluralLabel": "Duplicate Record Sets", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/DuplicateRecordItem" + }, + "DurableId": "DuplicateRecordItem", + "QualifiedApiName": "DuplicateRecordItem", + "DeveloperName": "DuplicateRecordItem", + "Label": "Duplicate Record Item", + "PluralLabel": "Duplicate Record Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AssetRelationship" + }, + "DurableId": "AssetRelationship", + "QualifiedApiName": "AssetRelationship", + "DeveloperName": "AssetRelationship", + "Label": "Asset Relationship", + "PluralLabel": "Asset Relationships", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/MessagingEndUser" + }, + "DurableId": "MessagingEndUser", + "QualifiedApiName": "MessagingEndUser", + "DeveloperName": "MessagingEndUser", + "Label": "Messaging User", + "PluralLabel": "Messaging Users", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ScorecardMetric" + }, + "DurableId": "ScorecardMetric", + "QualifiedApiName": "ScorecardMetric", + "DeveloperName": "ScorecardMetric", + "Label": "Scorecard Metric", + "PluralLabel": "Scorecard Metrics", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Scorecard" + }, + "DurableId": "Scorecard", + "QualifiedApiName": "Scorecard", + "DeveloperName": "Scorecard", + "Label": "Scorecard", + "PluralLabel": "Scorecards", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/MessagingSession" + }, + "DurableId": "MessagingSession", + "QualifiedApiName": "MessagingSession", + "DeveloperName": "MessagingSession", + "Label": "Messaging Session", + "PluralLabel": "Messaging Sessions", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ReturnOrder" + }, + "DurableId": "ReturnOrder", + "QualifiedApiName": "ReturnOrder", + "DeveloperName": "ReturnOrder", + "Label": "Return Order", + "PluralLabel": "Return Orders", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ScorecardAssociation" + }, + "DurableId": "ScorecardAssociation", + "QualifiedApiName": "ScorecardAssociation", + "DeveloperName": "ScorecardAssociation", + "Label": "Scorecard Association", + "PluralLabel": "Scorecard Associations", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/RecordAction" + }, + "DurableId": "RecordAction", + "QualifiedApiName": "RecordAction", + "DeveloperName": "RecordAction", + "Label": "RecordAction", + "PluralLabel": "RecordActions", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ReturnOrderLineItem" + }, + "DurableId": "ReturnOrderLineItem", + "QualifiedApiName": "ReturnOrderLineItem", + "DeveloperName": "ReturnOrderLineItem", + "Label": "Return Order Line Item", + "PluralLabel": "Return Order Line Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactRequest" + }, + "DurableId": "ContactRequest", + "QualifiedApiName": "ContactRequest", + "DeveloperName": "ContactRequest", + "Label": "Contact Request", + "PluralLabel": "Contact Requests", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactPointAddress" + }, + "DurableId": "ContactPointAddress", + "QualifiedApiName": "ContactPointAddress", + "DeveloperName": "ContactPointAddress", + "Label": "Contact Point Address", + "PluralLabel": "Contact Point Addresses", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactPointEmail" + }, + "DurableId": "ContactPointEmail", + "QualifiedApiName": "ContactPointEmail", + "DeveloperName": "ContactPointEmail", + "Label": "Contact Point Email", + "PluralLabel": "Contact Point Emails", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactPointPhone" + }, + "DurableId": "ContactPointPhone", + "QualifiedApiName": "ContactPointPhone", + "DeveloperName": "ContactPointPhone", + "Label": "Contact Point Phone", + "PluralLabel": "Contact Point Phones", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkTypeGroup" + }, + "DurableId": "WorkTypeGroup", + "QualifiedApiName": "WorkTypeGroup", + "DeveloperName": "WorkTypeGroup", + "Label": "Work Type Group", + "PluralLabel": "Work Type Groups", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ConsumptionSchedule" + }, + "DurableId": "ConsumptionSchedule", + "QualifiedApiName": "ConsumptionSchedule", + "DeveloperName": "ConsumptionSchedule", + "Label": "Consumption Schedule", + "PluralLabel": "Consumption Schedules", + "InternalSharingModel": "Read", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ConsumptionRate" + }, + "DurableId": "ConsumptionRate", + "QualifiedApiName": "ConsumptionRate", + "DeveloperName": "ConsumptionRate", + "Label": "Consumption Rate", + "PluralLabel": "Consumption Rates", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ProductConsumptionSchedule" + }, + "DurableId": "ProductConsumptionSchedule", + "QualifiedApiName": "ProductConsumptionSchedule", + "DeveloperName": "ProductConsumptionSchedule", + "Label": "Product Consumption Schedule", + "PluralLabel": "Product Consumption Schedules", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ServiceTerritoryWorkType" + }, + "DurableId": "ServiceTerritoryWorkType", + "QualifiedApiName": "ServiceTerritoryWorkType", + "DeveloperName": "ServiceTerritoryWorkType", + "Label": "Service Territory Work Type", + "PluralLabel": "Service Territory Work Types", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PaymentGroup" + }, + "DurableId": "PaymentGroup", + "QualifiedApiName": "PaymentGroup", + "DeveloperName": "PaymentGroup", + "Label": "Payment Group", + "PluralLabel": "Payment Groups", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CardPaymentMethod" + }, + "DurableId": "CardPaymentMethod", + "QualifiedApiName": "CardPaymentMethod", + "DeveloperName": "CardPaymentMethod", + "Label": "Card Payment Method", + "PluralLabel": "Card Payment Methods", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactPointConsent" + }, + "DurableId": "ContactPointConsent", + "QualifiedApiName": "ContactPointConsent", + "DeveloperName": "ContactPointConsent", + "Label": "Contact Point Consent", + "PluralLabel": "Contact Point Consents", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ContactPointTypeConsent" + }, + "DurableId": "ContactPointTypeConsent", + "QualifiedApiName": "ContactPointTypeConsent", + "DeveloperName": "ContactPointTypeConsent", + "Label": "Contact Point Type Consent", + "PluralLabel": "Contact Point Type Consents", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/DataUseLegalBasis" + }, + "DurableId": "DataUseLegalBasis", + "QualifiedApiName": "DataUseLegalBasis", + "DeveloperName": "DataUseLegalBasis", + "Label": "Data Use Legal Basis", + "PluralLabel": "Data Use Legal Bases", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/DataUsePurpose" + }, + "DurableId": "DataUsePurpose", + "QualifiedApiName": "DataUsePurpose", + "DeveloperName": "DataUsePurpose", + "Label": "Data Use Purpose", + "PluralLabel": "Data Use Purposes", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Image" + }, + "DurableId": "Image", + "QualifiedApiName": "Image", + "DeveloperName": "Image", + "Label": "Image", + "PluralLabel": "Images", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PaymentAuthorization" + }, + "DurableId": "PaymentAuthorization", + "QualifiedApiName": "PaymentAuthorization", + "DeveloperName": "PaymentAuthorization", + "Label": "Payment Authorization", + "PluralLabel": "Payment Authorizations", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Recommendation" + }, + "DurableId": "Recommendation", + "QualifiedApiName": "Recommendation", + "DeveloperName": "Recommendation", + "Label": "Recommendation", + "PluralLabel": "Recommendations", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WebStore" + }, + "DurableId": "WebStore", + "QualifiedApiName": "WebStore", + "DeveloperName": "WebStore", + "Label": "Store", + "PluralLabel": "Stores", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkTypeGroupMember" + }, + "DurableId": "WorkTypeGroupMember", + "QualifiedApiName": "WorkTypeGroupMember", + "DeveloperName": "WorkTypeGroupMember", + "Label": "Work Type Group Member", + "PluralLabel": "Work Type Group Members", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AuthorizationForm" + }, + "DurableId": "AuthorizationForm", + "QualifiedApiName": "AuthorizationForm", + "DeveloperName": "AuthorizationForm", + "Label": "Authorization Form", + "PluralLabel": "Authorization Forms", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AuthorizationFormConsent" + }, + "DurableId": "AuthorizationFormConsent", + "QualifiedApiName": "AuthorizationFormConsent", + "DeveloperName": "AuthorizationFormConsent", + "Label": "Authorization Form Consent", + "PluralLabel": "Authorization Form Consents", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AuthorizationFormDataUse" + }, + "DurableId": "AuthorizationFormDataUse", + "QualifiedApiName": "AuthorizationFormDataUse", + "DeveloperName": "AuthorizationFormDataUse", + "Label": "Authorization Form Data Use", + "PluralLabel": "Authorization Form Data Uses", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AuthorizationFormText" + }, + "DurableId": "AuthorizationFormText", + "QualifiedApiName": "AuthorizationFormText", + "DeveloperName": "AuthorizationFormText", + "Label": "Authorization Form Text", + "PluralLabel": "Authorization Form Texts", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CartDeliveryGroup" + }, + "DurableId": "CartDeliveryGroup", + "QualifiedApiName": "CartDeliveryGroup", + "DeveloperName": "CartDeliveryGroup", + "Label": "Cart Delivery Group", + "PluralLabel": "Cart Delivery Groups", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CartItem" + }, + "DurableId": "CartItem", + "QualifiedApiName": "CartItem", + "DeveloperName": "CartItem", + "Label": "Cart Item", + "PluralLabel": "Cart Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/InvoiceLine" + }, + "DurableId": "InvoiceLine", + "QualifiedApiName": "InvoiceLine", + "DeveloperName": "InvoiceLine", + "Label": "Invoice Line", + "PluralLabel": "Invoice Lines", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PaymentGateway" + }, + "DurableId": "PaymentGateway", + "QualifiedApiName": "PaymentGateway", + "DeveloperName": "PaymentGateway", + "Label": "Payment Gateway", + "PluralLabel": "Payment Gateways", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PaymentLineInvoice" + }, + "DurableId": "PaymentLineInvoice", + "QualifiedApiName": "PaymentLineInvoice", + "DeveloperName": "PaymentLineInvoice", + "Label": "Payment Line Invoice", + "PluralLabel": "Payment Line Invoices", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PaymentMethod" + }, + "DurableId": "PaymentMethod", + "QualifiedApiName": "PaymentMethod", + "DeveloperName": "PaymentMethod", + "Label": "Payment Method", + "PluralLabel": "Payment Methods", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Refund" + }, + "DurableId": "Refund", + "QualifiedApiName": "Refund", + "DeveloperName": "Refund", + "Label": "Refund", + "PluralLabel": "Refunds", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Shift" + }, + "DurableId": "Shift", + "QualifiedApiName": "Shift", + "DeveloperName": "Shift", + "Label": "Shift", + "PluralLabel": "Shifts", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WebCart" + }, + "DurableId": "WebCart", + "QualifiedApiName": "WebCart", + "DeveloperName": "WebCart", + "Label": "Cart", + "PluralLabel": "Carts", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AssetStatePeriod" + }, + "DurableId": "AssetStatePeriod", + "QualifiedApiName": "AssetStatePeriod", + "DeveloperName": "AssetStatePeriod", + "Label": "Asset State Period", + "PluralLabel": "Asset State Periods", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CreditMemo" + }, + "DurableId": "CreditMemo", + "QualifiedApiName": "CreditMemo", + "DeveloperName": "CreditMemo", + "Label": "Credit Memo", + "PluralLabel": "Credit Memos", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CreditMemoLine" + }, + "DurableId": "CreditMemoLine", + "QualifiedApiName": "CreditMemoLine", + "DeveloperName": "CreditMemoLine", + "Label": "Credit Memo Line", + "PluralLabel": "Credit Memo Lines", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/RefundLinePayment" + }, + "DurableId": "RefundLinePayment", + "QualifiedApiName": "RefundLinePayment", + "DeveloperName": "RefundLinePayment", + "Label": "Refund Line Payment", + "PluralLabel": "Refund Line Payments", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AssetAction" + }, + "DurableId": "AssetAction", + "QualifiedApiName": "AssetAction", + "DeveloperName": "AssetAction", + "Label": "Asset Action", + "PluralLabel": "Asset Actions", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AssetActionSource" + }, + "DurableId": "AssetActionSource", + "QualifiedApiName": "AssetActionSource", + "DeveloperName": "AssetActionSource", + "Label": "Asset Action Source", + "PluralLabel": "Asset Action Sources", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CartTax" + }, + "DurableId": "CartTax", + "QualifiedApiName": "CartTax", + "DeveloperName": "CartTax", + "Label": "Cart Tax", + "PluralLabel": "Cart Taxes", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CommSubscription" + }, + "DurableId": "CommSubscription", + "QualifiedApiName": "CommSubscription", + "DeveloperName": "CommSubscription", + "Label": "Communication Subscription", + "PluralLabel": "Communication Subscriptions", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CommSubscriptionChannelType" + }, + "DurableId": "CommSubscriptionChannelType", + "QualifiedApiName": "CommSubscriptionChannelType", + "DeveloperName": "CommSubscriptionChannelType", + "Label": "Communication Subscription Channel Type", + "PluralLabel": "Communication Subscription Channel Types", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CommSubscriptionConsent" + }, + "DurableId": "CommSubscriptionConsent", + "QualifiedApiName": "CommSubscriptionConsent", + "DeveloperName": "CommSubscriptionConsent", + "Label": "Communication Subscription Consent", + "PluralLabel": "Communication Subscription Consents", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CommSubscriptionTiming" + }, + "DurableId": "CommSubscriptionTiming", + "QualifiedApiName": "CommSubscriptionTiming", + "DeveloperName": "CommSubscriptionTiming", + "Label": "Communication Subscription Timing", + "PluralLabel": "Communication Subscription Timings", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CredentialStuffingEventStore" + }, + "DurableId": "CredentialStuffingEventStore", + "QualifiedApiName": "CredentialStuffingEventStore", + "DeveloperName": "CredentialStuffingEventStore", + "Label": "Credential Stuffing Event Store", + "PluralLabel": "Credential Stuffing Event Stores", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CreditMemoInvApplication" + }, + "DurableId": "CreditMemoInvApplication", + "QualifiedApiName": "CreditMemoInvApplication", + "DeveloperName": "CreditMemoInvApplication", + "Label": "Credit Memo Invoice Application", + "PluralLabel": "Credit Memo Invoice Applications", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/DigitalWallet" + }, + "DurableId": "DigitalWallet", + "QualifiedApiName": "DigitalWallet", + "DeveloperName": "DigitalWallet", + "Label": "Digital Wallet", + "PluralLabel": "Digital Wallets", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/EngagementChannelType" + }, + "DurableId": "EngagementChannelType", + "QualifiedApiName": "EngagementChannelType", + "DeveloperName": "EngagementChannelType", + "Label": "Engagement Channel Type", + "PluralLabel": "Engagement Channel Types", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/FinanceBalanceSnapshot" + }, + "DurableId": "FinanceBalanceSnapshot", + "QualifiedApiName": "FinanceBalanceSnapshot", + "DeveloperName": "FinanceBalanceSnapshot", + "Label": "Finance Balance Snapshot", + "PluralLabel": "Finance Balance Snapshots", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/FinanceTransaction" + }, + "DurableId": "FinanceTransaction", + "QualifiedApiName": "FinanceTransaction", + "DeveloperName": "FinanceTransaction", + "Label": "Finance Transaction", + "PluralLabel": "Finance Transactions", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LegalEntity" + }, + "DurableId": "LegalEntity", + "QualifiedApiName": "LegalEntity", + "DeveloperName": "LegalEntity", + "Label": "Legal Entity", + "PluralLabel": "Legal Entities", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PartyConsent" + }, + "DurableId": "PartyConsent", + "QualifiedApiName": "PartyConsent", + "DeveloperName": "PartyConsent", + "Label": "Party Consent", + "PluralLabel": "Party Consents", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ReportAnomalyEventStore" + }, + "DurableId": "ReportAnomalyEventStore", + "QualifiedApiName": "ReportAnomalyEventStore", + "DeveloperName": "ReportAnomalyEventStore", + "Label": "Report Anomaly Event Store", + "PluralLabel": "Report Anomaly Event Stores", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/SessionHijackingEventStore" + }, + "DurableId": "SessionHijackingEventStore", + "QualifiedApiName": "SessionHijackingEventStore", + "DeveloperName": "SessionHijackingEventStore", + "Label": "Session Hijacking Event Store", + "PluralLabel": "Session Hijacking Event Stores", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkPlan" + }, + "DurableId": "WorkPlan", + "QualifiedApiName": "WorkPlan", + "DeveloperName": "WorkPlan", + "Label": "Work Plan", + "PluralLabel": "Work Plans", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkPlanTemplate" + }, + "DurableId": "WorkPlanTemplate", + "QualifiedApiName": "WorkPlanTemplate", + "DeveloperName": "WorkPlanTemplate", + "Label": "Work Plan Template", + "PluralLabel": "Work Plan Templates", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkPlanTemplateEntry" + }, + "DurableId": "WorkPlanTemplateEntry", + "QualifiedApiName": "WorkPlanTemplateEntry", + "DeveloperName": "WorkPlanTemplateEntry", + "Label": "Work Plan Template Entry", + "PluralLabel": "Work Plan Template Entries", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkStep" + }, + "DurableId": "WorkStep", + "QualifiedApiName": "WorkStep", + "DeveloperName": "WorkStep", + "Label": "Work Step", + "PluralLabel": "Work Steps", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WorkStepTemplate" + }, + "DurableId": "WorkStepTemplate", + "QualifiedApiName": "WorkStepTemplate", + "DeveloperName": "WorkStepTemplate", + "Label": "Work Step Template", + "PluralLabel": "Work Step Templates", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ApiAnomalyEventStore" + }, + "DurableId": "ApiAnomalyEventStore", + "QualifiedApiName": "ApiAnomalyEventStore", + "DeveloperName": "ApiAnomalyEventStore", + "Label": "API Anomaly Event Store", + "PluralLabel": "API Anomaly Event Stores", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LocWaitlistMsgTemplate" + }, + "DurableId": "LocWaitlistMsgTemplate", + "QualifiedApiName": "LocWaitlistMsgTemplate", + "DeveloperName": "LocWaitlistMsgTemplate", + "Label": "Queue Messaging Template", + "PluralLabel": "Queue Messaging Template", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LocationGroup" + }, + "DurableId": "LocationGroup", + "QualifiedApiName": "LocationGroup", + "DeveloperName": "LocationGroup", + "Label": "Location Group", + "PluralLabel": "Location Groups", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LocationGroupAssignment" + }, + "DurableId": "LocationGroupAssignment", + "QualifiedApiName": "LocationGroupAssignment", + "DeveloperName": "LocationGroupAssignment", + "Label": "Location Group Assignment", + "PluralLabel": "Location Group Assignments", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LocationWaitlist" + }, + "DurableId": "LocationWaitlist", + "QualifiedApiName": "LocationWaitlist", + "DeveloperName": "LocationWaitlist", + "Label": "Queue", + "PluralLabel": "Queues", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/LocationWaitlistedParty" + }, + "DurableId": "LocationWaitlistedParty", + "QualifiedApiName": "LocationWaitlistedParty", + "DeveloperName": "LocationWaitlistedParty", + "Label": "Queued Party", + "PluralLabel": "Queued Parties", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/OperatingHoursHoliday" + }, + "DurableId": "OperatingHoursHoliday", + "QualifiedApiName": "OperatingHoursHoliday", + "DeveloperName": "OperatingHoursHoliday", + "Label": "Operating Hours Holiday", + "PluralLabel": "Operating Hours Holidays", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ProcessException" + }, + "DurableId": "ProcessException", + "QualifiedApiName": "ProcessException", + "DeveloperName": "ProcessException", + "Label": "Process Exception", + "PluralLabel": "Process Exceptions", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ReturnOrderItemAdjustment" + }, + "DurableId": "ReturnOrderItemAdjustment", + "QualifiedApiName": "ReturnOrderItemAdjustment", + "DeveloperName": "ReturnOrderItemAdjustment", + "Label": "Return Order Item Adjustment", + "PluralLabel": "Return Order Item Adjustments", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ReturnOrderItemTax" + }, + "DurableId": "ReturnOrderItemTax", + "QualifiedApiName": "ReturnOrderItemTax", + "DeveloperName": "ReturnOrderItemTax", + "Label": "Return Order Item Tax", + "PluralLabel": "Return Order Item Taxes", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AlternativePaymentMethod" + }, + "DurableId": "AlternativePaymentMethod", + "QualifiedApiName": "AlternativePaymentMethod", + "DeveloperName": "AlternativePaymentMethod", + "Label": "Alternative Payment Method", + "PluralLabel": "Alternative Payment Methods", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/PaymentAuthAdjustment" + }, + "DurableId": "PaymentAuthAdjustment", + "QualifiedApiName": "PaymentAuthAdjustment", + "DeveloperName": "PaymentAuthAdjustment", + "Label": "Payment Authorization Adjustment", + "PluralLabel": "Payment Authorization Adjustments", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/AppointmentTopicTimeSlot" + }, + "DurableId": "AppointmentTopicTimeSlot", + "QualifiedApiName": "AppointmentTopicTimeSlot", + "DeveloperName": "AppointmentTopicTimeSlot", + "Label": "Appointment Topic Time Slot", + "PluralLabel": "Appointment Topic Time Slots", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "ReadWrite", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/BusinessBrand" + }, + "DurableId": "BusinessBrand", + "QualifiedApiName": "BusinessBrand", + "DeveloperName": "BusinessBrand", + "Label": "Business Brand", + "PluralLabel": "Business Brands", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/CaseRelatedIssue" + }, + "DurableId": "CaseRelatedIssue", + "QualifiedApiName": "CaseRelatedIssue", + "DeveloperName": "CaseRelatedIssue", + "Label": "Case Related Issue", + "PluralLabel": "Case Related Issues", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ChangeRequest" + }, + "DurableId": "ChangeRequest", + "QualifiedApiName": "ChangeRequest", + "DeveloperName": "ChangeRequest", + "Label": "Change Request", + "PluralLabel": "Change Requests", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ChangeRequestRelatedIssue" + }, + "DurableId": "ChangeRequestRelatedIssue", + "QualifiedApiName": "ChangeRequestRelatedIssue", + "DeveloperName": "ChangeRequestRelatedIssue", + "Label": "Change Request Related Issue", + "PluralLabel": "Change Request Related Issues", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Customer" + }, + "DurableId": "Customer", + "QualifiedApiName": "Customer", + "DeveloperName": "Customer", + "Label": "Customer", + "PluralLabel": "Customers", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Incident" + }, + "DurableId": "Incident", + "QualifiedApiName": "Incident", + "DeveloperName": "Incident", + "Label": "Incident", + "PluralLabel": "Incidents", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Problem" + }, + "DurableId": "Problem", + "QualifiedApiName": "Problem", + "DeveloperName": "Problem", + "Label": "Problem", + "PluralLabel": "Problems", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ProblemIncident" + }, + "DurableId": "ProblemIncident", + "QualifiedApiName": "ProblemIncident", + "DeveloperName": "ProblemIncident", + "Label": "Related Problem and Incident", + "PluralLabel": "Related Problems and Incidents", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Seller" + }, + "DurableId": "Seller", + "QualifiedApiName": "Seller", + "DeveloperName": "Seller", + "Label": "Seller", + "PluralLabel": "Sellers", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ChangeRequestRelatedItem" + }, + "DurableId": "ChangeRequestRelatedItem", + "QualifiedApiName": "ChangeRequestRelatedItem", + "DeveloperName": "ChangeRequestRelatedItem", + "Label": "Change Request Related Item", + "PluralLabel": "Change Request Related Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/IncidentRelatedItem" + }, + "DurableId": "IncidentRelatedItem", + "QualifiedApiName": "IncidentRelatedItem", + "DeveloperName": "IncidentRelatedItem", + "Label": "Incident Related Item", + "PluralLabel": "Incident Related Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/ProblemRelatedItem" + }, + "DurableId": "ProblemRelatedItem", + "QualifiedApiName": "ProblemRelatedItem", + "DeveloperName": "ProblemRelatedItem", + "Label": "Problem Related Item", + "PluralLabel": "Problem Related Items", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/WebCartAdjustmentBasis" + }, + "DurableId": "WebCartAdjustmentBasis", + "QualifiedApiName": "WebCartAdjustmentBasis", + "DeveloperName": "WebCartAdjustmentBasis", + "Label": "Cart Adjustment Basis", + "PluralLabel": "Cart Adjustment Bases", + "InternalSharingModel": "ControlledByParent", + "ExternalSharingModel": "ControlledByParent", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/Activity" + }, + "DurableId": "Activity", + "QualifiedApiName": "Activity", + "DeveloperName": "Activity", + "Label": "Activity", + "PluralLabel": "Activities", + "InternalSharingModel": "Private", + "ExternalSharingModel": "Private", + "DeploymentStatus": null + }, + { + "attributes": { + "type": "EntityDefinition", + "url": "/services/data/v54.0/tooling/sobjects/EntityDefinition/01I5i000000Y6fp" + }, + "DurableId": "01I5i000000Y6fp", + "QualifiedApiName": "Property__c", + "DeveloperName": "Property", + "Label": "Property", + "PluralLabel": "Properties", + "InternalSharingModel": "ReadWrite", + "ExternalSharingModel": "Private", + "DeploymentStatus": "Deployed" + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/property_custom_fields_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/property_custom_fields_soql_response.json new file mode 100644 index 00000000000000..fb8a815a37d441 --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/property_custom_fields_soql_response.json @@ -0,0 +1,81 @@ +{ + "size": 3, + "totalSize": 3, + "done": true, + "queryLocator": null, + "entityTypeName": "CustomField", + "records": [ + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004hSOmEAM" + }, + "DeveloperName": "Price", + "CreatedDate": "2022-05-04T11:01:45.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-04T11:01:45.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004jLgGEAU" + }, + "DeveloperName": "Description", + "CreatedDate": "2022-05-17T10:44:17.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": "more details about property", + "LastModifiedDate": "2022-05-17T10:44:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + }, + { + "attributes": { + "type": "CustomField", + "url": "/services/data/v54.0/tooling/sobjects/CustomField/00N5i000004jN1REAU" + }, + "DeveloperName": "Active", + "CreatedDate": "2022-05-17T12:26:33.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "InlineHelpText": null, + "LastModifiedDate": "2022-05-17T12:26:33.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/property_custom_object_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/property_custom_object_soql_response.json new file mode 100644 index 00000000000000..6b0fb98e96043f --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/property_custom_object_soql_response.json @@ -0,0 +1,34 @@ +{ + "size": 1, + "totalSize": 1, + "done": true, + "queryLocator": null, + "entityTypeName": "CustomObject", + "records": [ + { + "attributes": { + "type": "CustomObject", + "url": "/services/data/v54.0/tooling/sobjects/CustomObject/01I5i000000Y6fpEAC" + }, + "Description": "A tangible asset such as an apartment, a house, etc", + "Language": "en_US", + "ManageableState": "unmanaged", + "CreatedDate": "2022-05-04T10:59:12.000+0000", + "CreatedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "LastModifiedDate": "2022-05-17T10:40:43.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + } + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/property_fields_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/property_fields_soql_response.json new file mode 100644 index 00000000000000..23c3777733bec2 --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/property_fields_soql_response.json @@ -0,0 +1,544 @@ +{ + "size": 15, + "totalSize": 15, + "done": true, + "queryLocator": null, + "entityTypeName": "EntityParticle", + "records": [ + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.Id" + }, + "QualifiedApiName": "Id", + "DeveloperName": "Id", + "Label": "Record ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.Id" + }, + "DataType": "Lookup()", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "id", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.Owner" + }, + "QualifiedApiName": "OwnerId", + "DeveloperName": "Owner", + "Label": "Owner ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.Owner" + }, + "DataType": "Lookup(User,Group)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "Group", + "User" + ] + }, + "RelationshipName": "Owner", + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.IsDeleted" + }, + "QualifiedApiName": "IsDeleted", + "DeveloperName": "IsDeleted", + "Label": "Deleted", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.IsDeleted" + }, + "DataType": "Checkbox", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "boolean", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.Name" + }, + "QualifiedApiName": "Name", + "DeveloperName": "Name", + "Label": "Property Name", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.Name" + }, + "DataType": "Text(80)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "string", + "Precision": 0, + "Scale": 0, + "Length": 80, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.CreatedDate" + }, + "QualifiedApiName": "CreatedDate", + "DeveloperName": "CreatedDate", + "Label": "Created Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.CreatedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.CreatedBy" + }, + "QualifiedApiName": "CreatedById", + "DeveloperName": "CreatedBy", + "Label": "Created By ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.CreatedBy" + }, + "DataType": "Lookup(User)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "User" + ] + }, + "RelationshipName": "CreatedBy", + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.LastModifiedDate" + }, + "QualifiedApiName": "LastModifiedDate", + "DeveloperName": "LastModifiedDate", + "Label": "Last Modified Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.LastModifiedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.LastModifiedBy" + }, + "QualifiedApiName": "LastModifiedById", + "DeveloperName": "LastModifiedBy", + "Label": "Last Modified By ID", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.LastModifiedBy" + }, + "DataType": "Lookup(User)", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "reference", + "Precision": 0, + "Scale": 0, + "Length": 18, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": [ + "User" + ] + }, + "RelationshipName": "LastModifiedBy", + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.SystemModstamp" + }, + "QualifiedApiName": "SystemModstamp", + "DeveloperName": "SystemModstamp", + "Label": "System Modstamp", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.SystemModstamp" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": true, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.LastActivityDate" + }, + "QualifiedApiName": "LastActivityDate", + "DeveloperName": "LastActivityDate", + "Label": "Last Activity Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.LastActivityDate" + }, + "DataType": "Date", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "date", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.LastViewedDate" + }, + "QualifiedApiName": "LastViewedDate", + "DeveloperName": "LastViewedDate", + "Label": "Last Viewed Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.LastViewedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.LastReferencedDate" + }, + "QualifiedApiName": "LastReferencedDate", + "DeveloperName": "LastReferencedDate", + "Label": "Last Referenced Date", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.LastReferencedDate" + }, + "DataType": "Date/Time", + "LastModifiedDate": null, + "LastModifiedBy": null, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "datetime", + "Precision": 0, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.00N5i000004hSOm" + }, + "QualifiedApiName": "Price__c", + "DeveloperName": "Price", + "Label": "Price", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.00N5i000004hSOm" + }, + "DataType": "Currency(18, 0)", + "LastModifiedDate": "2022-05-04T11:01:45.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "currency", + "Precision": 18, + "Scale": 0, + "Length": 0, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.00N5i000004jLgG" + }, + "QualifiedApiName": "Description__c", + "DeveloperName": "Description", + "Label": "Description", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.00N5i000004jLgG" + }, + "DataType": "Text Area(255)", + "LastModifiedDate": "2022-05-17T10:44:17.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "textarea", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": true + }, + { + "attributes": { + "type": "EntityParticle", + "url": "/services/data/v54.0/tooling/sobjects/EntityParticle/01I5i000000Y6fp.00N5i000004jN1R" + }, + "QualifiedApiName": "Active__c", + "DeveloperName": "Active", + "Label": "Active", + "FieldDefinition": { + "attributes": { + "type": "FieldDefinition", + "url": "/services/data/v54.0/tooling/sobjects/FieldDefinition/01I5i000000Y6fp.00N5i000004jN1R" + }, + "DataType": "Picklist", + "LastModifiedDate": "2022-05-17T12:26:33.000+0000", + "LastModifiedBy": { + "attributes": { + "type": "User", + "url": "/services/data/v54.0/tooling/sobjects/User/0055i000001Vi2IAAS" + }, + "Username": "user@mydomain.com" + }, + "IsIndexed": false, + "ComplianceGroup": null, + "SecurityClassification": null + }, + "DataType": "picklist", + "Precision": 0, + "Scale": 0, + "Length": 255, + "Digits": 0, + "IsUnique": false, + "IsCompound": false, + "IsComponent": false, + "ReferenceTo": { + "referenceTo": null + }, + "RelationshipName": null, + "IsNillable": false + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/record_count_property_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/record_count_property_response.json new file mode 100644 index 00000000000000..dd193cc507081c --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/record_count_property_response.json @@ -0,0 +1,8 @@ +{ + "sObjects": [ + { + "count": 3, + "name": "Property__c" + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/versions_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/versions_response.json new file mode 100644 index 00000000000000..13dfd022c51b8a --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/versions_response.json @@ -0,0 +1,172 @@ +[ + { + "label": "Spring '11", + "url": "/services/data/v21.0", + "version": "21.0" + }, + { + "label": "Summer '11", + "url": "/services/data/v22.0", + "version": "22.0" + }, + { + "label": "Winter '12", + "url": "/services/data/v23.0", + "version": "23.0" + }, + { + "label": "Spring '12", + "url": "/services/data/v24.0", + "version": "24.0" + }, + { + "label": "Summer '12", + "url": "/services/data/v25.0", + "version": "25.0" + }, + { + "label": "Winter '13", + "url": "/services/data/v26.0", + "version": "26.0" + }, + { + "label": "Spring '13", + "url": "/services/data/v27.0", + "version": "27.0" + }, + { + "label": "Summer '13", + "url": "/services/data/v28.0", + "version": "28.0" + }, + { + "label": "Winter '14", + "url": "/services/data/v29.0", + "version": "29.0" + }, + { + "label": "Spring '14", + "url": "/services/data/v30.0", + "version": "30.0" + }, + { + "label": "Summer '14", + "url": "/services/data/v31.0", + "version": "31.0" + }, + { + "label": "Winter '15", + "url": "/services/data/v32.0", + "version": "32.0" + }, + { + "label": "Spring '15", + "url": "/services/data/v33.0", + "version": "33.0" + }, + { + "label": "Summer '15", + "url": "/services/data/v34.0", + "version": "34.0" + }, + { + "label": "Winter '16", + "url": "/services/data/v35.0", + "version": "35.0" + }, + { + "label": "Spring '16", + "url": "/services/data/v36.0", + "version": "36.0" + }, + { + "label": "Summer '16", + "url": "/services/data/v37.0", + "version": "37.0" + }, + { + "label": "Winter '17", + "url": "/services/data/v38.0", + "version": "38.0" + }, + { + "label": "Spring '17", + "url": "/services/data/v39.0", + "version": "39.0" + }, + { + "label": "Summer '17", + "url": "/services/data/v40.0", + "version": "40.0" + }, + { + "label": "Winter '18", + "url": "/services/data/v41.0", + "version": "41.0" + }, + { + "label": "Spring ’18", + "url": "/services/data/v42.0", + "version": "42.0" + }, + { + "label": "Summer '18", + "url": "/services/data/v43.0", + "version": "43.0" + }, + { + "label": "Winter '19", + "url": "/services/data/v44.0", + "version": "44.0" + }, + { + "label": "Spring '19", + "url": "/services/data/v45.0", + "version": "45.0" + }, + { + "label": "Summer '19", + "url": "/services/data/v46.0", + "version": "46.0" + }, + { + "label": "Winter '20", + "url": "/services/data/v47.0", + "version": "47.0" + }, + { + "label": "Spring '20", + "url": "/services/data/v48.0", + "version": "48.0" + }, + { + "label": "Summer '20", + "url": "/services/data/v49.0", + "version": "49.0" + }, + { + "label": "Winter '21", + "url": "/services/data/v50.0", + "version": "50.0" + }, + { + "label": "Spring '21", + "url": "/services/data/v51.0", + "version": "51.0" + }, + { + "label": "Summer '21", + "url": "/services/data/v52.0", + "version": "52.0" + }, + { + "label": "Winter '22", + "url": "/services/data/v53.0", + "version": "53.0" + }, + { + "label": "Spring '22", + "url": "/services/data/v54.0", + "version": "54.0" + } +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json new file mode 100644 index 00000000000000..6cb938aaa04c69 --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json @@ -0,0 +1,192 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {\"Durable Id\": \"Account\", \"Qualified API Name\": \"Account\", \"Developer Name\": \"Account\", \"Label\": \"Account\", \"Plural Label\": \"Accounts\", \"Internal Sharing Model\": \"ReadWrite\", \"External Sharing Model\": \"Private\"}, \"name\": \"Account\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"\", \"platform\": \"urn:li:dataPlatform:salesforce\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.OtherSchema\": {\"rawSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"Id\", \"nullable\": false, \"description\": \"Account ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup()\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"IsDeleted\", \"nullable\": false, \"description\": \"Deleted\", \"type\": {\"type\": {\"com.linkedin.schema.BooleanType\": {}}}, \"nativeDataType\": \"Checkbox\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"MasterRecordId\", \"nullable\": true, \"description\": \"Master Record ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(Account)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Name\", \"nullable\": false, \"description\": \"Account Name\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Name\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CompareName\", \"nullable\": true, \"description\": \"Compare Name\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(80)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Type\", \"nullable\": true, \"description\": \"Account Type\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ParentId\", \"nullable\": true, \"description\": \"Parent Account ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Hierarchy\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingStreet\", \"nullable\": true, \"description\": \"Billing Street\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingCity\", \"nullable\": true, \"description\": \"Billing City\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingState\", \"nullable\": true, \"description\": \"Billing State/Province\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingPostalCode\", \"nullable\": true, \"description\": \"Billing Zip/Postal Code\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingCountry\", \"nullable\": true, \"description\": \"Billing Country\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingLatitude\", \"nullable\": true, \"description\": \"Billing Latitude\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingLongitude\", \"nullable\": true, \"description\": \"Billing Longitude\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"BillingGeocodeAccuracy\", \"nullable\": true, \"description\": \"Billing Geocode Accuracy\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingStreet\", \"nullable\": true, \"description\": \"Shipping Street\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingCity\", \"nullable\": true, \"description\": \"Shipping City\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingState\", \"nullable\": true, \"description\": \"Shipping State/Province\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingPostalCode\", \"nullable\": true, \"description\": \"Shipping Zip/Postal Code\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingCountry\", \"nullable\": true, \"description\": \"Shipping Country\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingLatitude\", \"nullable\": true, \"description\": \"Shipping Latitude\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingLongitude\", \"nullable\": true, \"description\": \"Shipping Longitude\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ShippingGeocodeAccuracy\", \"nullable\": true, \"description\": \"Shipping Geocode Accuracy\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Address\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Phone\", \"nullable\": true, \"description\": \"Account Phone\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Phone\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Fax\", \"nullable\": true, \"description\": \"Account Fax\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Fax\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"AccountNumber\", \"nullable\": true, \"description\": \"Account Number\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(40)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Website\", \"nullable\": true, \"description\": \"Website\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"URL(255)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"PhotoUrl\", \"nullable\": true, \"description\": \"Photo URL\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"URL(255)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Sic\", \"nullable\": true, \"description\": \"SIC Code\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(20)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Industry\", \"nullable\": true, \"description\": \"Industry\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"AnnualRevenue\", \"nullable\": true, \"description\": \"Annual Revenue\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Currency(18, 0)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"NumberOfEmployees\", \"nullable\": true, \"description\": \"Employees\", \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Number(8, 0)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Ownership\", \"nullable\": true, \"description\": \"Ownership\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"TickerSymbol\", \"nullable\": true, \"description\": \"Ticker Symbol\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Content(20)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Description\", \"nullable\": true, \"description\": \"Account Description\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Long Text Area(32000)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Rating\", \"nullable\": true, \"description\": \"Account Rating\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Site\", \"nullable\": true, \"description\": \"Account Site\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(80)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CompareSite\", \"nullable\": true, \"description\": \"Compare Site\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(80)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"OwnerId\", \"nullable\": false, \"description\": \"Owner ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(User)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"OwnerAlias\", \"nullable\": true, \"description\": \"Owner Alias\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(30)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CreatedDate\", \"nullable\": false, \"description\": \"Created Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CreatedById\", \"nullable\": false, \"description\": \"Created By ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(User)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastModifiedDate\", \"nullable\": false, \"description\": \"Last Modified Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastModifiedById\", \"nullable\": false, \"description\": \"Last Modified By ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(User)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"SystemModstamp\", \"nullable\": false, \"description\": \"System Modstamp\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastActivityDate\", \"nullable\": true, \"description\": \"Last Activity\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastViewedDate\", \"nullable\": true, \"description\": \"Last Viewed Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastReferencedDate\", \"nullable\": true, \"description\": \"Last Referenced Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Jigsaw\", \"nullable\": true, \"description\": \"Data.com Key\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(20)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"JigsawCompanyId\", \"nullable\": true, \"description\": \"Jigsaw Company ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"External Lookup\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CleanStatus\", \"nullable\": true, \"description\": \"Clean Status\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"AccountSource\", \"nullable\": true, \"description\": \"Account Source\", \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"DunsNumber\", \"nullable\": true, \"description\": \"D-U-N-S Number\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(9)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Tradestyle\", \"nullable\": true, \"description\": \"Tradestyle\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(255)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"NaicsCode\", \"nullable\": true, \"description\": \"NAICS Code\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(8)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"NaicsDesc\", \"nullable\": true, \"description\": \"NAICS Description\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(120)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"YearStarted\", \"nullable\": true, \"description\": \"Year Started\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(4)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"SicDesc\", \"nullable\": true, \"description\": \"SIC Description\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(80)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"DandbCompanyId\", \"nullable\": true, \"description\": \"D&B Company ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(D&B Company)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"OperatingHoursId\", \"nullable\": true, \"description\": \"Operating Hour ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(Operating Hours)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ConnectionReceivedDate\", \"nullable\": true, \"description\": \"Received Connection Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"ConnectionSentDate\", \"nullable\": true, \"description\": \"Sent Connection Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Tier\", \"nullable\": true, \"description\": \"Einstein Account Tier\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(2)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CustomerPriority__c\", \"nullable\": true, \"description\": \"Customer Priority\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"SLA__c\", \"nullable\": true, \"description\": \"SLA\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Active__c\", \"nullable\": true, \"description\": \"Active\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"NumberofLocations__c\", \"nullable\": true, \"description\": \"Number of Locations\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Number(3, 0)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"UpsellOpportunity__c\", \"nullable\": true, \"description\": \"Upsell Opportunity\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"SLASerialNumber__c\", \"nullable\": true, \"description\": \"SLA Serial Number\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(10)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"SLAExpirationDate__c\", \"nullable\": true, \"description\": \"SLA Expiration Date\", \"created\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651461737000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}], \"primaryKeys\": [\"Id\"], \"foreignKeys\": [{\"name\": \"MasterRecord\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),MasterRecordId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD)\"}, {\"name\": \"Parent\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),ParentId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD)\"}, {\"name\": \"Owner\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),OwnerId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD)\"}, {\"name\": \"CreatedBy\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),CreatedById)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD)\"}, {\"name\": \"LastModifiedBy\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),LastModifiedById)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD)\"}, {\"name\": \"DandbCompany\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,DandBCompany,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),DandbCompanyId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,DandBCompany,PROD)\"}, {\"name\": \"OperatingHours\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,OperatingHours,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD),OperatingHoursId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,OperatingHours,PROD)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Account,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Standard Object\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1651661952000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:user@mydomain.com\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1651661952000}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1652784043000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:user@mydomain.com\", \"operationType\": \"ALTER\", \"lastUpdatedTimestamp\": 1652784043000}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {\"Durable Id\": \"01I5i000000Y6fp\", \"Qualified API Name\": \"Property__c\", \"Developer Name\": \"Property\", \"Label\": \"Property\", \"Plural Label\": \"Properties\", \"Internal Sharing Model\": \"ReadWrite\", \"External Sharing Model\": \"Private\", \"Language\": \"en_US\", \"Manageable State\": \"unmanaged\"}, \"name\": \"Property\", \"description\": \"A tangible asset such as an apartment, a house, etc\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"\", \"platform\": \"urn:li:dataPlatform:salesforce\", \"version\": 0, \"created\": {\"time\": 1651661952000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.OtherSchema\": {\"rawSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"Id\", \"nullable\": false, \"description\": \"Record ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup()\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"OwnerId\", \"nullable\": false, \"description\": \"Owner ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(User,Group)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"IsDeleted\", \"nullable\": false, \"description\": \"Deleted\", \"type\": {\"type\": {\"com.linkedin.schema.BooleanType\": {}}}, \"nativeDataType\": \"Checkbox\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Name\", \"nullable\": true, \"description\": \"Property Name\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text(80)\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CreatedDate\", \"nullable\": false, \"description\": \"Created Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"CreatedById\", \"nullable\": false, \"description\": \"Created By ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(User)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastModifiedDate\", \"nullable\": false, \"description\": \"Last Modified Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastModifiedById\", \"nullable\": false, \"description\": \"Last Modified By ID\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Lookup(User)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"SystemModstamp\", \"nullable\": false, \"description\": \"System Modstamp\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:SystemField\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastActivityDate\", \"nullable\": true, \"description\": \"Last Activity Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastViewedDate\", \"nullable\": true, \"description\": \"Last Viewed Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"LastReferencedDate\", \"nullable\": true, \"description\": \"Last Referenced Date\", \"type\": {\"type\": {\"com.linkedin.schema.DateType\": {}}}, \"nativeDataType\": \"Date/Time\", \"recursive\": false, \"globalTags\": {\"tags\": []}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Price__c\", \"nullable\": false, \"description\": \"Price\", \"created\": {\"time\": 1651662105000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1651662105000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.NumberType\": {}}}, \"nativeDataType\": \"Currency(18, 0)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Description__c\", \"nullable\": true, \"description\": \"Description\", \"created\": {\"time\": 1652784257000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1652784257000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"Text Area(255)\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}, {\"fieldPath\": \"Active__c\", \"nullable\": false, \"description\": \"Active\", \"created\": {\"time\": 1652790393000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"lastModified\": {\"time\": 1652790393000, \"actor\": \"urn:li:corpuser:user@mydomain.com\"}, \"type\": {\"type\": {\"com.linkedin.schema.EnumType\": {}}}, \"nativeDataType\": \"Picklist\", \"recursive\": false, \"globalTags\": {\"tags\": [{\"tag\": \"urn:li:tag:Custom\"}]}, \"isPartOfKey\": false, \"jsonProps\": \"{}\"}], \"primaryKeys\": [\"Id\"], \"foreignKeys\": [{\"name\": \"Owner\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Group,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD),OwnerId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,Group,PROD)\"}, {\"name\": \"Owner\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD),OwnerId)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD)\"}, {\"name\": \"CreatedBy\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD),CreatedById)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD)\"}, {\"name\": \"LastModifiedBy\", \"foreignFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD),Id)\"], \"sourceFields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD),LastModifiedById)\"], \"foreignDataset\": \"urn:li:dataset:(urn:li:dataPlatform:salesforce,User,PROD)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Custom Object\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:sales\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:salesforce,Property__c,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1652353200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 3, \"columnCount\": 15}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1652353200000, + "runId": "salesforce-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/salesforce/test_salesforce.py b/metadata-ingestion/tests/integration/salesforce/test_salesforce.py new file mode 100644 index 00000000000000..6b3d8c1f7dc141 --- /dev/null +++ b/metadata-ingestion/tests/integration/salesforce/test_salesforce.py @@ -0,0 +1,114 @@ +import json +import pathlib +from unittest import mock + +from freezegun import freeze_time + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + +FROZEN_TIME = "2022-05-12 11:00:00" + +test_resources_dir = None + + +def _read_response(file_name: str) -> dict: + response_json_path = f"{test_resources_dir}/mock_files/{file_name}" + with open(response_json_path) as file: + data = json.loads(file.read()) + return data + + +def side_effect_call_salesforce(type, url): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + if url.endswith("/services/data/"): + return MockResponse(_read_response("versions_response.json"), 200) + if url.endswith("FROM EntityDefinition WHERE IsCustomizable = true"): + return MockResponse(_read_response("entity_definition_soql_response.json"), 200) + elif url.endswith("FROM EntityParticle WHERE EntityDefinitionId='Account'"): + return MockResponse(_read_response("account_fields_soql_response.json"), 200) + elif url.endswith("FROM CustomField WHERE EntityDefinitionId='Account'"): + return MockResponse( + _read_response("account_custom_fields_soql_response.json"), 200 + ) + elif url.endswith("FROM CustomObject where DeveloperName='Property'"): + return MockResponse( + _read_response("property_custom_object_soql_response.json"), 200 + ) + elif url.endswith( + "FROM EntityParticle WHERE EntityDefinitionId='01I5i000000Y6fp'" + ): # DurableId of Property__c + return MockResponse(_read_response("property_fields_soql_response.json"), 200) + elif url.endswith("FROM CustomField WHERE EntityDefinitionId='01I5i000000Y6fp'"): + return MockResponse( + _read_response("property_custom_fields_soql_response.json"), 200 + ) + elif url.endswith("/recordCount?sObjects=Property__c"): + return MockResponse(_read_response("record_count_property_response.json"), 200) + return MockResponse({}, 404) + + +@freeze_time(FROZEN_TIME) +def test_salesforce_ingest(pytestconfig, tmp_path): + + global test_resources_dir + test_resources_dir = pathlib.Path( + pytestconfig.rootpath / "tests/integration/salesforce" + ) + + with mock.patch("simple_salesforce.Salesforce") as mock_sdk: + mock_sf = mock.Mock() + mocked_call = mock.Mock() + mocked_call.side_effect = side_effect_call_salesforce + mock_sf._call_salesforce = mocked_call + mock_sdk.return_value = mock_sf + + pipeline = Pipeline.create( + { + "run_id": "salesforce-test", + "source": { + "type": "salesforce", + "config": { + "auth": "DIRECT_ACCESS_TOKEN", + "instance_url": "https://mydomain.my.salesforce.com/", + "access_token": "access_token`", + "ingest_tags": True, + "object_pattern": { + "allow": [ + "^Account$", + "^Property__c$", + ], + }, + "domain": {"sales": {"allow": {"^Property__c$"}}}, + "profiling": {"enabled": True}, + "profile_pattern": { + "allow": [ + "^Property__c$", + ] + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/salesforce_mces.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/salesforce_mces.json", + golden_path=test_resources_dir / "salesforce_mces_golden.json", + ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, + ) diff --git a/metadata-ingestion/tests/integration/tableau/setup/workbooksConnection_all.json b/metadata-ingestion/tests/integration/tableau/setup/workbooksConnection_all.json index 308e9b3e021a3f..5de093ae3f927c 100644 --- a/metadata-ingestion/tests/integration/tableau/setup/workbooksConnection_all.json +++ b/metadata-ingestion/tests/integration/tableau/setup/workbooksConnection_all.json @@ -1522,6 +1522,24 @@ } ] }, + { + "__typename": "ColumnField", + "id": "xxxdc8ee-30e5-644b-cf76-48a3dea79cda", + "name": null, + "description": null, + "isHidden": false, + "folderName": null, + "dataCategory": "NOMINAL", + "role": "DIMENSION", + "dataType": "STRING", + "defaultFormat": null, + "aggregation": null, + "columns": [ + { + "table": {} + } + ] + }, { "__typename": "ColumnField", "id": "277dc8ee-30e5-644b-cf76-48a3dea79cda", diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index 22da4587d07689..db69d5cbdc365e 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -1,51205 +1,50565 @@ [ - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"1f15d897-7f0c-7c59-037a-afa6a9b7c9a9\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15995\", \"name\": \"Email Performance by Campaign\", \"description\": \"Description for Email Performance by Campaign\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"1f15d897-7f0c-7c59-037a-afa6a9b7c9a9\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15995\", \"name\": \"Email Performance by Campaign\", \"description\": \"Description for Email Performance by Campaign\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Workbook\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Name": "", - "Activity Date": "", - "ID": "", - "Program Name": "", - "Active": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", - "title": "Timeline - Sent", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign/Timeline - Sent" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Name": "", + "Activity Date": "", + "ID": "", + "Program Name": "", + "Active": "", + "Id": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline - Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign/Timeline - Sent" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", - "Name": "", - "Id (Activity - Click Email)": "", - "Activity Date": "", - "Clickthrough Rate": "formula: [Clickthrough Emails]/[Delivered Email]", - "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", - "Sent Email": "formula: COUNTD([Id])", - "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", - "ID": "", - "Id (Activity - Open Email)": "", - "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", - "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", - "Id (Activity - Email Delivered)": "", - "Program Name": "", - "Active": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", - "title": "Campaign List", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign/Campaign List" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", + "Name": "", + "Id (Activity - Click Email)": "", + "Activity Date": "", + "Clickthrough Rate": "formula: [Clickthrough Emails]/[Delivered Email]", + "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", + "Sent Email": "formula: COUNTD([Id])", + "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", + "ID": "", + "Id (Activity - Open Email)": "", + "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", + "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", + "Id (Activity - Email Delivered)": "", + "Program Name": "", + "Active": "", + "Id": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign/Campaign List" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", - "Name": "", - "Id (Activity - Click Email)": "", - "Activity Date": "", - "Clickthrough Rate": "formula: [Clickthrough Emails]/[Delivered Email]", - "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", - "Measure Names": "", - "Sent Email": "formula: COUNTD([Id])", - "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", - "ID": "", - "Id (Activity - Open Email)": "", - "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", - "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", - "Id (Activity - Email Delivered)": "", - "Program Name": "", - "Measure Values": "", - "Active": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", - "title": "Summary", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign/Summary" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", + "Name": "", + "Id (Activity - Click Email)": "", + "Activity Date": "", + "Clickthrough Rate": "formula: [Clickthrough Emails]/[Delivered Email]", + "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", + "Measure Names": "", + "Sent Email": "formula: COUNTD([Id])", + "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", + "ID": "", + "Id (Activity - Open Email)": "", + "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", + "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", + "Id (Activity - Email Delivered)": "", + "Program Name": "", + "Measure Values": "", + "Active": "", + "Id": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign/Summary" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", - "Name": "", - "Id (Activity - Click Email)": "", - "Activity Date": "", - "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", - "Sent Email": "formula: COUNTD([Id])", - "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", - "ID": "", - "Id (Activity - Open Email)": "", - "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", - "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", - "Id (Activity - Email Delivered)": "", - "Program Name": "", - "Id": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", - "title": "Mobile - Sent by Campaign", - "description": "", - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign/Mobile - Sent by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Delivery Rate": "formula: ZN([Delivered Email]/[Sent Email])", + "Name": "", + "Id (Activity - Click Email)": "", + "Activity Date": "", + "Open Rate": "formula: ZN([Opened Email]/[Delivered Email])", + "Sent Email": "formula: COUNTD([Id])", + "Delivered Email": "formula: COUNTD([Id (Activity - Email Delivered)])", + "ID": "", + "Id (Activity - Open Email)": "", + "Clickthrough Emails": "formula: COUNTD([Id (Activity - Click Email)])", + "Click-to-Open": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "Opened Email": "formula: COUNTD([Id (Activity - Open Email)])", + "Id (Activity - Email Delivered)": "", + "Program Name": "", + "Id": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile - Sent by Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign/Mobile - Sent by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "externalUrl": null, - "title": "Email Performance by Campaign", - "description": "", - "charts": [ - "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", - "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", - "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", - "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" - ], - "lastModified": { - "created": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1640200234000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "externalUrl": null, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign", - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Email Performance by Campaign/Email Performance by Campaign" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign", + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Email Performance by Campaign/Marketo" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-02-09T00:05:25Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-02-09T00:05:25Z", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "Marketo", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "2018-02-09T00:05:25Z", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "Marketo", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "jsonPath": null, + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Program ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Delivery Rate", - "jsonPath": null, - "nullable": false, - "description": "formula: ZN([Delivered Email]/[Sent Email])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Program ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Platform (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated At", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Workspace Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing ID (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign ID (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number of Records", - "jsonPath": null, - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign Run ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User Agent (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice Number (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign ID (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity Date (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has Predictive (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step ID (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Id (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step ID (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Clickthrough Rate", - "jsonPath": null, - "nullable": false, - "description": "formula: [Clickthrough Emails]/[Delivered Email]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Open Rate", - "jsonPath": null, - "nullable": false, - "description": "formula: ZN([Opened Email]/[Delivered Email])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Names", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sent Email", - "jsonPath": null, - "nullable": false, - "description": "formula: COUNTD([Id])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivered Email", - "jsonPath": null, - "nullable": false, - "description": "formula: COUNTD([Id (Activity - Email Delivered)])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead ID (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is Mobile Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing ID (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead ID (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test Variant (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign Run ID (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Id (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity Date (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign Run ID (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created At", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Clickthrough Emails", - "jsonPath": null, - "nullable": false, - "description": "formula: COUNTD([Id (Activity - Click Email)])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is Mobile Device (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing ID (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice Number (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Click-to-Open", - "jsonPath": null, - "nullable": false, - "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened Email", - "jsonPath": null, - "nullable": false, - "description": "formula: COUNTD([Id (Activity - Open Email)])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign Run ID (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened Non Clicked Emails", - "jsonPath": null, - "nullable": false, - "description": "formula: [Opened Email]-[Clickthrough Emails]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test Variant (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step ID (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Non Clickthrough Email", - "jsonPath": null, - "nullable": false, - "description": "formula: [Delivered Email]-[Clickthrough Emails]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead ID (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Id (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Program Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign ID (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity Date (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test Variant (Activity - Click Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Non Opened Email", - "jsonPath": null, - "nullable": false, - "description": "formula: [Delivered Email]-[Opened Email]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Values", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has Predictive (Activity - Email Delivered)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Bounceback", - "jsonPath": null, - "nullable": false, - "description": "formula: [Sent Email] - [Delivered Email]", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice Number (Activity - Open Email)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + { + "fieldPath": "Platform (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated At", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Workspace Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number of Records", + "jsonPath": null, + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign Run ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Is Predictive", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Id (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Clickthrough Rate", + "jsonPath": null, + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Open Rate", + "jsonPath": null, + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Names", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sent Email", + "jsonPath": null, + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivered Email", + "jsonPath": null, + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Is Mobile Device", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Platform", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Id (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created At", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Clickthrough Emails", + "jsonPath": null, + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Device (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Test Variant", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Link ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Click-to-Open", + "jsonPath": null, + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened Email", + "jsonPath": null, + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Link", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened Non Clicked Emails", + "jsonPath": null, + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Non Clickthrough Email", + "jsonPath": null, + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Device", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Program Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Has Predictive", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Non Opened Email", + "jsonPath": null, + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Values", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User Agent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Bounceback", + "jsonPath": null, + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity6" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"661fabd0-bed6-8610-e066-0694a81a6cea\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15619\", \"name\": \"Dvdrental Workbook\", \"description\": \"\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Workbook\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Custom SQL Query": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook/Sheet 1" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity11" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "payment_date": "", + "amount": "", + "Custom SQL Query": "", + "First Name": "", + "customer_id": "", + "rental_id": "", + "Last Name": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook/Sheet 2" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity10" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Category": "", + "Segment": "", + "Customer Name": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook/Sheet 3" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TAGSHEET3", + "context": null + } + ] + } + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity7" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "externalUrl": null, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard", + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook/dvd Rental Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/campaignsTable" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "externalUrl": null, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "programName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "programId", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "workspaceName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "updatedAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + "deleted": null + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1", + "access": null, + "lastRefreshed": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"661fabd0-bed6-8610-e066-0694a81a6cea\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15619\", \"name\": \"Dvdrental Workbook\", \"description\": \"\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Custom SQL Query": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", - "title": "Sheet 1", - "description": "", - "lastModified": { - "created": { - "time": 1639772911000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook/Sheet 1" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook/Story 1" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "payment_date": "", - "amount": "", - "Custom SQL Query": "", - "First Name": "", - "customer_id": "", - "rental_id": "", - "Last Name": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", - "title": "Sheet 2", - "description": "", - "lastModified": { - "created": { - "time": 1639773415000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook/Sheet 2" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Category": "", - "Segment": "", - "Customer Name": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", - "title": "Sheet 3", - "description": "", - "lastModified": { - "created": { - "time": 1640375456000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook/Sheet 3" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "False", + "extractLastRefreshTime": "", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "Customer Payment Query", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:TAGSHEET3" - } - ] - } + }, + "fields": [ + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:YEAR", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Custom SQL Query", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "First Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Last Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "externalUrl": null, - "title": "dvd Rental Dashboard", - "description": "", - "charts": [ - "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" - ], - "lastModified": { - "created": { - "time": 1639773866000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard", - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook/dvd Rental Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.address,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.actor,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "False", + "extractLastRefreshTime": "", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "actor+ (dvdrental)", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Last Update", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "District", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "First Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Postal Code", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "country, city", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Phone", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Last Update (Address)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Address Id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Actor Id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "actor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Last Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "City Id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] } }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "externalUrl": null, - "title": "Story 1", - "description": "", - "charts": [], - "lastModified": { - "created": { - "time": 1639773866000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1642199995000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1", - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Dvdrental Workbook/Story 1" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "False", + "extractLastRefreshTime": "", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "Superstore Datasource", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Product ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Profit", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Product", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sub-Category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Row ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Customer ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Ship Mode", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "People", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order ID (Returns)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "City", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sales", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Region", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Ship Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Orders", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Profit Ratio", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Returns", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Manufacturer", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Returned", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Regional Manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Postal Code", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Profit (bin)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Segment", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Top Customers by Profit", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Country/Region", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Discount", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Product Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Customer Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] } }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Marketo/campaignsTable" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "Customer Payment Query", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + "fields": [ + { + "fieldPath": "programName", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Custom SQL Query", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "First Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Last Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + { + "fieldPath": "programId", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "createdAt", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "workspaceName", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "updatedAt", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"6ffa5a7f-d852-78f1-6c6d-20ac23610ebf\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15605\", \"name\": \"Executive Dashboard\", \"description\": \"\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.address,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.actor,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Workbook\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "actor+ (dvdrental)", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Opened": "", + "Total # Request": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "Due date": "", + "Priority": "", + "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "Active": "", + "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Total Active Requests": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Number": "", + "Closed": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Address2", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Last Update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "District", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "First Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Postal Code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "country, city", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:HIERARCHYFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Phone", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Last Update (Address)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Address Id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Actor Id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "actor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Last Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "City Id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Opened Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Dvdrental Workbook/Superstore Datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "Superstore Datasource", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Opened": "", + "Due date": "", + "Priority": "", + "Name": "", + "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "Active": "", + "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Number": "", + "Closed": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Region (People)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Product ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Profit", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Product", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sub-Category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Row ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Customer ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ship Mode", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "People", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order ID (Returns)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "City", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sales", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Region", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ship Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Orders", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Profit Ratio", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Returns", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Manufacturer", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Returned", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Regional Manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Postal Code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Profit (bin)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Segment", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Top Customers by Profit", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Country/Region", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Discount", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Product Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Customer Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Top 10 Items by Requests and YoY Change" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity6" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Overdue": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "Number": "", + "Priority": "", + "Due date": "", + "Total # Problems": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Closed": "", + "Active": "", + "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Total Active Problems": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Opened Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity11" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "% of Overdue": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "Priority": "", + "Category (Incident)": "", + "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "Number": "", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Due date": "", + "Active": "", + "Closed": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Overdue" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity10" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Number": "", + "Priority": "", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "% of critical and high priority": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "Active": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/High and Critical Priority Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity7" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Priority": "", + "Current Year Total Cases": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Category (Incident)": "", + "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "Number": "", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Due date": "", + "Active": "", + "Closed": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Total Incidents by Category and YoY Change" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/campaignsTable" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Number": "", + "Priority": "", + "Problems with Related Incidents": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "Related Incidents": "", + "Opened": "", + "% of known error": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Known error": "", + "Active": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", + "title": "Known Errors", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "programName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "programId", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "workspaceName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "updatedAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Known Errors" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)/address" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Opened": "", + "Due date": "", + "Priority": "", + "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "Active": "", + "% of Overdue": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "Number": "", + "Closed": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I2", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Overdue Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)/actor" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Priority": "", + "Category (Incident)": "", + "Time to Close an Incident (seconds)": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Active": "", + "Closed": "", + "Time to Close an Incident": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/AVG Time to Solve an Incident" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"6ffa5a7f-d852-78f1-6c6d-20ac23610ebf\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/15605\", \"name\": \"Executive Dashboard\", \"description\": \"\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Total # Request": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "Due date": "", - "Priority": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "Active": "", - "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Total Active Requests": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Number": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Requests", - "title": "Opened Requests", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Opened Requests" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Opened": "", + "% made SLA": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "Priority": "", + "Active": "", + "Made SLA": "", + "Number": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Made SLA?" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Due date": "", - "Priority": "", - "Name": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "Active": "", - "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Number": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top 10 Items by Requests and YoY Change", - "title": "Top 10 Items by Requests and YoY Change", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Top 10 Items by Requests and YoY Change" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Priority": "", + "Number": "", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Active": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Tooltip - Incident Breakdown by Priority" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Overdue": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Priority": "", - "Due date": "", - "Total # Problems": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Closed": "", - "Active": "", - "Current Year Total Cases": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Total Active Problems": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Problems", - "title": "Opened Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Opened Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Overdue": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "Number": "", + "Priority": "", + "Due date": "", + "% of Overdue": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Closed": "", + "Active": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Overdue Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "% of Overdue": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "Priority": "", - "Category (Incident)": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Due date": "", - "Active": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", - "title": "Overdue", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Overdue" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Number": "", + "Priority": "", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Active": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "% of critical and high priority": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High and Critical Priority Problems", - "title": "High and Critical Priority Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/High and Critical Priority Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" } - } + ], + "type": null, + "access": null, + "lastRefreshed": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Current Year Total Cases": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Category (Incident)": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Due date": "", - "Active": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total Incidents by Category and YoY Change", - "title": "Total Incidents by Category and YoY Change", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Total Incidents by Category and YoY Change" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Tooltip - Problem Breakdown by Priority" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Problems with Related Incidents": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", - "Related Incidents": "", - "Opened": "", - "% of known error": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Known error": "", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known Errors", - "title": "Known Errors", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Known Errors" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Due date": "", - "Priority": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "Active": "", - "% of Overdue": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "Number": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Requests", - "title": "Overdue Requests", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Overdue Requests" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Opened": "", + "Priority": "", + "Active": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Number": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Category (Incident)": "", - "Time to Close an Incident (seconds)": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Active": "", - "Closed": "", - "Time to Close an Incident": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG Time to Solve an Incident", - "title": "AVG Time to Solve an Incident", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/AVG Time to Solve an Incident" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" } - } + ], + "type": null, + "access": null, + "lastRefreshed": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "% made SLA": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "Priority": "", - "Active": "", - "Made SLA": "", - "Number": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made SLA?", - "title": "Made SLA?", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Made SLA?" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Tooltip - Request Breakdown by Priority" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Active": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", - "title": "Tooltip - Incident Breakdown by Priority", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Tooltip - Incident Breakdown by Priority" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Overdue": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Number": "", - "Priority": "", - "Due date": "", - "% of Overdue": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Closed": "", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue Problems", - "title": "Overdue Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Overdue Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Number": "", + "Priority": "", + "Time Span Breakdown": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Age of Problems": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "Active": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Active": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", - "title": "Tooltip - Problem Breakdown by Priority", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Tooltip - Problem Breakdown by Priority" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], + }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" } - } + ], + "type": null, + "access": null, + "lastRefreshed": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Opened": "", - "Priority": "", - "Active": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Number": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", - "title": "Tooltip - Request Breakdown by Priority", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Tooltip - Request Breakdown by Priority" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Age of Active Problems" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Number": "", - "Priority": "", - "Time Span Breakdown": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Age of Problems": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", - "Active": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age of Active Problems", - "title": "Age of Active Problems", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Age of Active Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Priority": "", - "Current Year Total Cases": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Category (Incident)": "", - "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "Total Active Incidents": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "Number": "", - "Opened": "", - "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "Opened Month Tooltip": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", - "Due date": "", - "Active": "", - "Closed": "" - }, - "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", - "title": "Opened Incidents", - "description": "", - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Opened Incidents" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "Priority": "", + "Current Year Total Cases": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Category (Incident)": "", + "Overdue": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "Total Active Incidents": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "Number": "", + "Opened": "", + "Max Year?": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "Opened Month Tooltip": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "Due date": "", + "Active": "", + "Closed": "" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, - "externalUrl": null, - "title": "Executive Dashboard", - "description": "", - "charts": [ - "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", - "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", - "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", - "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", - "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", - "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", - "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", - "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", - "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", - "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", - "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", - "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", - "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" - ], - "lastModified": { - "created": { - "time": 1639768450000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1639768502000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null }, - "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard", - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Executive Dashboard/Executive Dashboard" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" } - } + ], + "type": null, + "access": null, + "lastRefreshed": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Executive Dashboard/Requests" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Opened Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-01-18T19:35:39Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T19:35:39Z", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "Requests", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Closed by (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mobile Picture", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "% made SLA", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Meta", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Recurring price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Total # Request", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Requested for date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ordered item link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Visible elsewhere", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Recurring Price Frequency (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Update name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Special instructions", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Execution Plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Visible on Bundles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "No search", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "List Price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Customer update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Price (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order Guide", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Package", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Estimated Delivery", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Model", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Billable", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Recurring Price Frequency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Recurring Price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Fulfillment group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Backordered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "No cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ignore price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Overdue", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Names", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Replace on upgrade", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Context", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Requested for", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "No order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Billable (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "No quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mobile Picture Type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Protection policy", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Omit price in cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Catalogs", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Entitlement script", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Published version", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Preview link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Price (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Stage (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Visible on Guides", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item (Requested Item) 1", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery time", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Stage", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Request", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Current Year Total Cases", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Resolve Time", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Max Year?", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Icon", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Display name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number of Records", - "jsonPath": null, - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "No proceed checkout", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Values", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "SLA due (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Use cart layout", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "No order now", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "SLA due (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Application", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Template", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Catalog", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Image", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "% of Overdue", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Workflow", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Class", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created from item design", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Migrated Data", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed by (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Roles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "SLA due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Hide price (mobile listings)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Availability", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Vendor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Picture", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan script", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Requested Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Total Active Requests", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Start closed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Request state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service (Request)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Cost", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Catalog Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Executive Dashboard/Problems" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-01-18T20:21:33Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:21:33Z", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "Problems", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "externalUrl": null, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", "impersonator": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "SLA due (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Default assignee", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Workaround", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Overdue", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed by (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Migrated Data", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "u", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time Span Breakdown", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "u_", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Change request", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "% of Overdue", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "SLA due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Cost center", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Roles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Total # Problems", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number of Records", - "jsonPath": null, - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Exclude manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lucha", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Problems with Related Incidents", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Related Incidents", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Source", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "% of known error", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Problem state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Max Year?", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Known error", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group email", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Age of Problems", - "jsonPath": null, - "nullable": false, - "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Values", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "% of critical and high priority", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Include members", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Names", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Current Year Total Cases", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group (Problem)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Total Active Problems", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Group)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "deleted": null + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard", + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Executive Dashboard/Incidents" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "True", - "extractLastRefreshTime": "2018-01-18T20:13:08Z", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "2018-01-18T20:13:08Z", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "Incidents", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "2018-01-18T19:35:39Z", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "Requests", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Assignment group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Attributes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User input (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close code (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Migrated Data", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Status", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Notify (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Requires verification", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Maintenance schedule", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Warranty expiration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "GL account", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional assignee list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "First discovered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Asset", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "% of Overdue", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "SLA due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Skip sync", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "DNS Domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Caller (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Department", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Resolved by (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Current Year Total Cases", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Close notes (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Managed by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Names", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Model number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "PO number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Short description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business resolve time (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Child Incidents (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "IP Address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Asset tag", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due in", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Manufacturer", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Checked out", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Category (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Fully qualified domain name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Installed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Purchased", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lease contract", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Vendor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Overdue", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Category (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent Incident (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Cost currency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "SLA due (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Impact (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Subcategory (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened by (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Cost", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Monitor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Serial number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Model ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Parent (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time to Close an Incident (seconds)", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Owned by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Invoice number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated by (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval set (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Start date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ordered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business duration (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order received", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Discovery source", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed by (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Total Active Incidents", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - }, - { - "tag": "urn:li:tag:ATTRIBUTE" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Class", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Operational status", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Expected start (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work notes list (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Resolve time (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reopen count (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created by (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned to (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Most recent discovery", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon reject (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Knowledge (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Max Year?", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Watch list (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Fault count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Caused by Change (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updated (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "MAC Address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Priority (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Company (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Additional comments (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Business service (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Schedule", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Supported by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Configuration item (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Support group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation display (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Justification", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Change Request (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Updates", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Incident state (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Made SLA (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Opened Month Tooltip", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Problem (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Measure Values", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Group list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Checked in", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Severity (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number of Records", - "jsonPath": null, - "nullable": false, - "description": "formula: 1", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time worked (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Cost center", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work end (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Due date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Created (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Delivery plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Subcategory (Configuration Item)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sys ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments and Work notes (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Can Print", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "BOOLEAN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Number (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Follow up (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Task type (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Domain Path (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Closed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Description (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Reassignment count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Contact type (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assignment group (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Work start (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Correlation ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Resolved (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Assigned", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Time to Close an Incident", - "jsonPath": null, - "nullable": false, - "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Upon approval (Incident)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity6" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Opened", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Mobile Picture", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity11" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Delivery task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "% made SLA", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity10" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Meta", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Correlation ID (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity7" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Domain Path (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Work notes (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/campaignsTable" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Short description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "programName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "programId", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "workspaceName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "updatedAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)/address" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Domain Path (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Updated by (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I2", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + { + "fieldPath": "Additional comments (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Recurring price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Total # Request", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Requested for date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Ordered item link", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Visible elsewhere", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Update name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Special instructions", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Execution Plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Visible on Bundles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "No search", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "List Price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Customer update", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Price (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order Guide", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Package", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Estimated Delivery", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Model", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Billable", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Recurring Price Frequency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Recurring Price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Fulfillment group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional comments (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Backordered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "No cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Ignore price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Overdue", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Names", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Replace on upgrade", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Context", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Requested for", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "No order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Billable (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "No quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mobile Picture Type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Protection policy", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Omit price in cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Catalogs", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Entitlement script", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Published version", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Preview link", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Price (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Stage (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Visible on Guides", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery time", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Stage", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery task (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Request", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Current Year Total Cases", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Resolve Time", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Max Year?", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Icon", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Display name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number of Records", + "jsonPath": null, + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "No proceed checkout", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Values", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "SLA due (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Use cart layout", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "No order now", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "SLA due (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Application", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Template", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Catalog", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Image", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "% of Overdue", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Workflow", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Class", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created from item design", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Migrated Data", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed by (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Roles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "SLA due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Hide price (mobile listings)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Availability", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Vendor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Picture", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan script", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Requested Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Total Active Requests", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery task (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Start closed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Request state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service (Request)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Cost", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain Path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Catalog Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "2018-01-18T20:21:33Z", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "Problems", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Default assignee", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Workaround", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Overdue", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain Path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed by (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Migrated Data", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery task (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "u", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time Span Breakdown", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "u_", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Change request", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "% of Overdue", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "SLA due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Cost center", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Roles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Total # Problems", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain Path (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number of Records", + "jsonPath": null, + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Exclude manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lucha", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Problems with Related Incidents", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Related Incidents", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Source", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "% of known error", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Problem state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Max Year?", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional comments (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Known error", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group email", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Age of Problems", + "jsonPath": null, + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Values", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "% of critical and high priority", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Include members", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Names", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Current Year Total Cases", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group (Problem)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Total Active Problems", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Group)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "2018-01-18T20:13:08Z", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "Incidents", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Attributes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User input (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close code (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Migrated Data", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery task (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Status", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Notify (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Requires verification", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Maintenance schedule", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Warranty expiration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "GL account", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional assignee list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "First discovered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Asset", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "% of Overdue", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "SLA due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Skip sync", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "DNS Domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Caller (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Department", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Resolved by (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Current Year Total Cases", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Close notes (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Managed by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Names", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Model number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "PO number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Short description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business resolve time (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Child Incidents (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "IP Address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Asset tag", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due in", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Manufacturer", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Checked out", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Category (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Fully qualified domain name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Installed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Purchased", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lease contract", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Vendor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Overdue", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Category (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent Incident (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Cost currency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "SLA due (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Impact (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Subcategory (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened by (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Cost", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Monitor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Serial number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Model ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Parent (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Owned by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Invoice number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated by (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval set (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Start date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Ordered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business duration (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Order received", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Discovery source", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed by (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Total Active Incidents", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + }, + { + "tag": "urn:li:tag:ATTRIBUTE", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Class", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Operational status", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Expected start (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work notes list (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Resolve time (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reopen count (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created by (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Most recent discovery", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon reject (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Knowledge (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Max Year?", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Location (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Watch list (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Fault count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Caused by Change (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updated (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "MAC Address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Priority (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Company (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Additional comments (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Business service (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Schedule", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Supported by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Configuration item (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Support group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation display (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Justification", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Change Request (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Updates", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Incident state (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Made SLA (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Opened Month Tooltip", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Problem (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Measure Values", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Group list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Checked in", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Severity (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number of Records", + "jsonPath": null, + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time worked (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Cost center", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work end (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain Path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Due date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Created (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Delivery plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Sys ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Can Print", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Number (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Follow up (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Task type (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Domain Path (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Closed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Description (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Reassignment count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Contact type (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assignment group (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Work start (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Correlation ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Resolved (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Assigned", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Time to Close an Incident", + "jsonPath": null, + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Upon approval (Incident)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Incidents/task" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Requests/sc_request" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "requested_for", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "requested_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "request_state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "stage", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "special_instructions", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_stc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Requests/sc_req_item" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "configuration_item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sc_catalog", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "stage", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "estimated_delivery", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "context", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "billable", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_frequency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cat_item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_guide", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "backordered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "request", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Requests/sc_cat_item" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mobile_picture_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "workflow", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_customer_update", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "visible_standalone", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_scope", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "template", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_proceed_checkout", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "billable", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "meta", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ordered_item_link", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sc_ic_version", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "image", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_policy", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "roles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "picture", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "list_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_order_now", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "vendor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sc_ic_item_staging", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "entitlement_script", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "icon", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ignore_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "start_closed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_package", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_replace_on_upgrade", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "use_sc_layout", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "availability", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "model", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "custom_cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mobile_picture", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mobile_hide_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_time", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_search", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_frequency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan_script", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "visible_bundle", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_update_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "visible_guide", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "preview", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "omit_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Problems/sys_user_group" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "source", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "exclude_manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost_center", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "u_lucha", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "u_lu2", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "roles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "default_assignee", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "email", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "include_members", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Problems/problem" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "related_incidents", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rfc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "problem_state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_around", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "known_error", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Incidents/incident" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "severity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent_incident", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "subcategory", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "caller_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "resolved_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "resolved_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_stc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "child_incidents", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "incident_state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "notify", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "problem_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "caused_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reopen_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_stc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rfc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_code", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Incidents/cmdb_ci" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "operational_status", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_discovered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost_cc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "checked_in", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "attributes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "serial_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "vendor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ip_address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "support_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "asset", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "supported_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "invoice_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "managed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "fault_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_in", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "justification", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "model_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "lease_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "monitor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost_center", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "subcategory", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "can_print", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "model_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "start_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "discovery_source", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "schedule", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "fqdn", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "warranty_expiration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "owned_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "asset_tag", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "manufacturer", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "purchase_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "department", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "checked_out", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "unverified", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "skip_sync", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "po_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gl_account", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "maintenance_schedule", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "install_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "dns_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mac_address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "change_control", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "install_status", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"bd040833-8f66-22c0-1b51-bd4ccf5eef7c\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/17904\", \"name\": \"Workbook published ds\", \"description\": \"\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Workbook\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "staff_last_name": "", + "amount": "", + "customer_first_name": "" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": null, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ], + "type": null, + "access": null, + "lastRefreshed": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Workbook published ds/published sheet ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "False", + "extractLastRefreshTime": "", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "", + "type": "EmbeddedDatasource" + }, + "externalUrl": null, + "name": "test publish datasource", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Custom SQL Query", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_last_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_first_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_last_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_first_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Embedded Data Source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Marketo/activity6" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_Run_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity_Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice_Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Has_Predictive", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Marketo/activity11" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_Run_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Link", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Test_Variant", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Platform", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity_Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice_Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Link_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Is_Predictive", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Device", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Is_Mobile_Device", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User_Agent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Marketo/activity10" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Device", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice_Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity_Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Test_Variant", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Is_Mobile_Device", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Has_Predictive", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "User_Agent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_Run_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Marketo/activity7" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_Run_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Activity_Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Mailing_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Has_Predictive", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Campaign_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Step_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Lead_ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Choice_Number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/actor+ (dvdrental)/address" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "phone", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address2", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "city_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "district", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "address_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/actor+ (dvdrental)/actor" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "actor_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Incidents/task" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Requests/sc_request" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "requested_for", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "requested_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "request_state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "stage", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "special_instructions", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_stc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)/actor" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Requests/sc_req_item" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "configuration_item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sc_catalog", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "stage", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "estimated_delivery", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "context", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "billable", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_frequency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cat_item", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_guide", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "backordered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "request", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Requests/sc_cat_item" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mobile_picture_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "workflow", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_customer_update", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "visible_standalone", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_scope", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "template", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_proceed_checkout", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "billable", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "meta", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ordered_item_link", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sc_ic_version", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "image", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_policy", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "roles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "picture", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "list_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_order_now", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "vendor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sc_ic_item_staging", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "entitlement_script", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "icon", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ignore_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "start_closed", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_package", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_replace_on_upgrade", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "use_sc_layout", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "availability", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "model", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "custom_cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mobile_picture", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_cart", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mobile_hide_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_time", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "no_search", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_frequency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "recurring_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan_script", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "visible_bundle", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_update_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "visible_guide", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "preview", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "omit_price", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Problems/sys_user_group" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "source", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "exclude_manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost_center", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "u_lucha", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "u_lu2", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "roles", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "default_assignee", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "email", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "manager", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "include_members", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Problems/problem" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "related_incidents", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rfc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "problem_state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents/task" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests/sc_request" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "requested_for", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "requested_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "request_state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "stage", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "special_instructions", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_stc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests/sc_req_item" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "configuration_item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sc_catalog", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "stage", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "estimated_delivery", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "context", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "billable", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_frequency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cat_item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order_guide", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "backordered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "request", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests/sc_cat_item" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "sc_catalogs", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mobile_picture_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "workflow", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_customer_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "visible_standalone", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_scope", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "template", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_proceed_checkout", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "billable", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "meta", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ordered_item_link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sc_ic_version", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "image", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_policy", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "roles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "picture", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "list_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_order_now", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "vendor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sc_ic_item_staging", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "entitlement_script", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "icon", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ignore_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "start_closed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_package", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_replace_on_upgrade", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_FLOAT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "use_sc_layout", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "availability", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "model", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "custom_cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mobile_picture", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mobile_hide_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_time", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_search", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_frequency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan_script", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "visible_bundle", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_update_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "visible_guide", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "preview", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "omit_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Problems/sys_user_group" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "u_u", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "source", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "exclude_manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost_center", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "u_lucha", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "u_lu2", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "roles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "default_assignee", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "include_members", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Problems/problem" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "related_incidents", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "rfc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "problem_state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_around", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "known_error", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents/incident" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "work_around", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "severity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent_incident", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "subcategory", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "caller_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "resolved_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "resolved_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_stc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "child_incidents", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "incident_state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "notify", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "problem_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "caused_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reopen_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_stc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "rfc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "known_error", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents/cmdb_ci" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Incidents/incident" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "first_discovered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "operational_status", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_discovered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost_cc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "checked_in", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "attributes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "serial_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "vendor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ip_address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "support_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "asset", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "supported_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "invoice_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "managed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "fault_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_in", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "justification", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "model_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "lease_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "monitor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost_center", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "subcategory", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "can_print", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "model_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "start_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "discovery_source", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "schedule", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "fqdn", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "warranty_expiration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "owned_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "asset_tag", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "manufacturer", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "purchase_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "department", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "checked_out", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "unverified", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "skip_sync", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "po_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "gl_account", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "maintenance_schedule", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "install_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "dns_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mac_address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "change_control", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "install_status", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + { + "fieldPath": "urgency", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "severity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_service", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_set", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent_incident", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "subcategory", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "caller_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "resolved_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "group_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_display", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "resolved_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_stc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "parent", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_input", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "escalation", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval_history", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "impact", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "expected_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "additional_assignee_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "child_incidents", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cmdb_ci", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "incident_state", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "notify", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reassignment_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "contact_type", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "problem_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "opened_at", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_notes_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "time_worked", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "caused_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_reject", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_task", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "knowledge", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "closed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_end", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "reopen_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "work_start", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "made_sla", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "calendar_stc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "rfc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_plan", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_code", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "close_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "activity_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "active", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "business_duration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "follow_up", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments_and_work_notes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "upon_approval", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "watch_list", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sla_due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"tableau\", \"workbook_id\": \"bd040833-8f66-22c0-1b51-bd4ccf5eef7c\"}, \"externalUrl\": \"https://do-not-connect/#/site/acryl/workbooks/17904\", \"name\": \"Workbook published ds\", \"description\": \"\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:tableau\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Workbook\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jawadqu@gmail.com\", \"type\": \"DATAOWNER\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "aspects": [ - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "staff_last_name": "", - "amount": "", - "customer_first_name": "" - }, - "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", - "title": "published sheet ds", - "description": "", - "lastModified": { - "created": { - "time": 1641951867000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "lastModified": { - "time": 1642658093000, - "actor": "urn:li:corpuser:jawadqu@gmail.com", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" - } - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/tableau/default/Workbook published ds/published sheet ds" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Incidents/cmdb_ci" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "operational_status", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_discovered", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost_cc", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "checked_in", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "attributes", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "serial_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "vendor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "ip_address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "support_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "asset", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "supported_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "invoice_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "managed_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "fault_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due_in", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "correlation_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "justification", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "model_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_class_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "comments", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "lease_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "monitor", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "cost_center", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "subcategory", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "delivery_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assignment_group", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "can_print", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "short_description", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "model_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "start_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "discovery_source", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_domain_path", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "assigned_to", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "schedule", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "fqdn", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "warranty_expiration", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "owned_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "asset_tag", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "manufacturer", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "purchase_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "department", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_updated_on", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "checked_out", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "unverified", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "skip_sync", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "po_number", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gl_account", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "maintenance_schedule", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "install_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "dns_domain", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_created_by", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "mac_address", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "change_control", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "install_status", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "due", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "sys_mod_count", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null } - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "chart", - "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Workbook published ds/test publish datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "EmbeddedDatasource" - }, - "externalUrl": null, - "name": "test publish datasource", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "False", + "extractLastRefreshTime": "", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "", + "type": "PublishedDatasource" + }, + "externalUrl": null, + "name": "test publish datasource", + "qualifiedName": null, + "description": "description for test publish datasource", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:YEAR", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "staff_first_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Custom SQL Query", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "staff_last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "staff_first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DATASOURCEFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "Published SQL Query", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_last_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_first_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_last_name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Embedded Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Published Data Source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore.xls.people,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore.xls.returns,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore.xls.orders,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity6" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity11" - ] + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "False", + "extractLastRefreshTime": "", + "extractLastIncrementalUpdateTime": "", + "extractLastUpdateTime": "", + "type": "PublishedDatasource" + }, + "externalUrl": null, + "name": "Superstore Datasource", + "qualifiedName": null, + "description": "Description for Superstore dataset", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Returns", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Link_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity10" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Segment", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Profit Ratio", + "jsonPath": null, + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Platform", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Is_Mobile_Device", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "User_Agent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/activity7" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "City", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Profit", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Test_Variant", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_Run_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Activity_Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Mailing_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Has_Predictive", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Campaign_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Step_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Lead_ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Choice_Number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Marketo/campaignsTable" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Quantity", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Returned", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "programName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "programId", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "createdAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "workspaceName", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "updatedAt", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)/address" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Product Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I2", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/actor+ (dvdrental)/actor" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Orders", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Product ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DBTIMESTAMP", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents/task" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Profit (bin)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Order ID (Returns)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests/sc_request" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Person", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Sub-Category", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "requested_for", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "requested_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "request_state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "stage", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "special_instructions", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_stc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests/sc_req_item" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Postal Code", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Product", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "configuration_item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sc_catalog", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "stage", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "estimated_delivery", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "context", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "billable", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_frequency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cat_item", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order_guide", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "backordered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "request", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Requests/sc_cat_item" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Ship Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:YEAR", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Location", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "sc_catalogs", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mobile_picture_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "workflow", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_customer_update", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "visible_standalone", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_scope", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "template", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_proceed_checkout", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "billable", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "meta", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ordered_item_link", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sc_ic_version", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "image", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_policy", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "roles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "picture", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "list_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_order_now", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "vendor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sc_ic_item_staging", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "entitlement_script", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "icon", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ignore_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "start_closed", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_package", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_replace_on_upgrade", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_FLOAT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "use_sc_layout", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "availability", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "model", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "custom_cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mobile_picture", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_cart", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mobile_hide_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_time", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "no_search", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_frequency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "recurring_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan_script", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "visible_bundle", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_update_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "visible_guide", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "preview", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "omit_price", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Problems/sys_user_group" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "People", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Country/Region", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "u_u", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "source", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "exclude_manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost_center", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "u_lucha", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "u_lu2", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "roles", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "default_assignee", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "manager", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "include_members", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Problems/problem" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Customer ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Region", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "related_incidents", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "rfc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "problem_state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_around", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "known_error", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents/incident" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Ship Mode", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Order ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:COUNT", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "urgency", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "severity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_service", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_set", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent_incident", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "subcategory", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "caller_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "resolved_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "group_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_display", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "resolved_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_stc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "parent", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "user_input", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "escalation", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval_history", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "impact", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "expected_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "additional_assignee_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "child_incidents", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cmdb_ci", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "incident_state", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "notify", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reassignment_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "contact_type", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "problem_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "opened_at", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_notes_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "priority", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "time_worked", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "caused_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_reject", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_task", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "knowledge", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "closed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_end", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "reopen_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "work_start", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "made_sla", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "calendar_stc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "rfc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_plan", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "close_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "activity_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "business_duration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "follow_up", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments_and_work_notes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "upon_approval", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "watch_list", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sla_due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Incidents/cmdb_ci" - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Sales", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Customer Name", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "first_discovered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "operational_status", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "last_discovered", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost_cc", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "checked_in", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "attributes", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "serial_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "vendor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ip_address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "support_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "asset", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "supported_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "invoice_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "managed_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "fault_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due_in", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "correlation_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "justification", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "model_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_class_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "comments", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "company", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "lease_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "monitor", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "cost_center", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "subcategory", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "delivery_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assignment_group", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "can_print", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "short_description", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "model_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "start_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "discovery_source", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_domain_path", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "assigned_to", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "schedule", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "fqdn", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "warranty_expiration", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "owned_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "asset_tag", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "manufacturer", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "purchase_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "WDC_DATE", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "department", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_updated_on", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "checked_out", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "unverified", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "skip_sync", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "WDC_BOOL", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "po_number", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "order_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "gl_account", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "maintenance_schedule", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "install_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "dns_domain", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_created_by", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "mac_address", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "change_control", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "WDC_STRING", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "install_status", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "due", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "WDC_DATETIME", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "sys_mod_count", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "WDC_INT", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/test publish datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, - "externalUrl": null, - "name": "test publish datasource", - "qualifiedName": null, - "description": "description for test publish datasource", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Row ID", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Manufacturer", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:GROUPFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "DATETIME", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "staff_first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Published SQL Query", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "customer_first_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "staff_last_name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Published Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore.xls.people,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore.xls.returns,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore.xls.orders,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/Samples/Superstore Datasource" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jawadqu@gmail.com", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "hasExtracts": "False", - "extractLastRefreshTime": "", - "extractLastIncrementalUpdateTime": "", - "extractLastUpdateTime": "", - "type": "PublishedDatasource" - }, - "externalUrl": null, - "name": "Superstore Datasource", - "qualifiedName": null, - "description": "Description for Superstore dataset", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Region (People)", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "Discount", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:SUM", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "Top Customers by Profit", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:SETFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Returns", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Segment", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Profit Ratio", - "jsonPath": null, - "nullable": false, - "description": "formula: SUM([Profit])/SUM([Sales])", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:CALCULATEDFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "City", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Profit", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Quantity", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Returned", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Product Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Orders", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Product ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Profit (bin)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:BINFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order ID (Returns)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Person", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sub-Category", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Postal Code", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Product", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:HIERARCHYFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ship Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Location", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } - }, - "nativeDataType": "UNKNOWN", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:HIERARCHYFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "People", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": null - } - } - }, - "nativeDataType": "TABLE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Country/Region", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Customer ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Region", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Ship Mode", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:COUNT" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Sales", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Customer Name", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Row ID", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Manufacturer", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:GROUPFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Region (People)", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Discount", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "REAL", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:MEASURE" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:SUM" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Order Date", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "DATE", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - }, - { - "tag": "urn:li:tag:YEAR" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "State", - "jsonPath": null, - "nullable": false, - "description": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STRING", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:DIMENSION" - }, - { - "tag": "urn:li:tag:COLUMNFIELD" - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } + { + "fieldPath": "Order Date", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + }, + { + "tag": "urn:li:tag:YEAR", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "State", + "jsonPath": null, + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION", + "context": null + }, + { + "tag": "urn:li:tag:COLUMNFIELD", + "context": null + } + ] + }, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Published Data Source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Published Data Source\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:fad3de4b86519c3edeb685215fe0bab1\"}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.customer,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.payment,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.customer,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.payment,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "I4", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/Customer Payment Query/Custom SQL Query" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "Custom SQL Query", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", - "viewLanguage": "SQL" - } + { + "fieldPath": "rental_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query/Custom SQL Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "Custom SQL Query", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"View\", \"Custom SQL\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"View\", \"Custom SQL\"]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.customer,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.payment,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.staff,PROD)\", \"type\": \"TRANSFORMED\"}]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.customer,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.payment,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.staff,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test", - "platform": "urn:li:dataPlatform:tableau", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + { + "fieldPath": "staff_first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "customer_last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "STR", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/prod/tableau/default/test publish datasource/Custom SQL Query" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": {}, - "externalUrl": null, - "name": "Custom SQL Query", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", - "viewLanguage": "SQL" - } + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "staff_last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource/Custom SQL Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "Custom SQL Query", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] } }, - { - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"View\", \"Custom SQL\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1638860400000, - "runId": "tableau-test", - "registryName": null, - "registryVersion": null, - "properties": null - } + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"View\", \"Custom SQL\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "registryName": null, + "registryVersion": null, + "properties": null } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau.py b/metadata-ingestion/tests/integration/tableau/test_tableau.py index 06f28abf83a341..d87e8d8dc2f248 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau.py @@ -2,6 +2,7 @@ import pathlib from unittest import mock +import pytest from freezegun import freeze_time from datahub.ingestion.run.pipeline import Pipeline @@ -40,6 +41,7 @@ def side_effect_query_metadata(query): @freeze_time(FROZEN_TIME) +@pytest.mark.slow_unit def test_tableau_ingest(pytestconfig, tmp_path): global test_resources_dir @@ -69,7 +71,7 @@ def test_tableau_ingest(pytestconfig, tmp_path): "connect_uri": "https://do-not-connect", "site": "acryl", "projects": ["default", "Project 2"], - "workbooks_page_size": 10, + "page_size": 10, "ingest_tags": True, "ingest_owner": True, "ingest_tables_external": True, diff --git a/metadata-ingestion/tests/integration/trino/test_trino.py b/metadata-ingestion/tests/integration/trino/test_trino.py index e5f76f13015377..594672f87ea0f1 100644 --- a/metadata-ingestion/tests/integration/trino/test_trino.py +++ b/metadata-ingestion/tests/integration/trino/test_trino.py @@ -51,6 +51,7 @@ def loaded_trino(trino_runner): @freeze_time(FROZEN_TIME) +@pytest.mark.xfail # TODO: debug the flakes for this test @pytest.mark.integration def test_trino_ingest( loaded_trino, test_resources_dir, pytestconfig, tmp_path, mock_time diff --git a/metadata-ingestion/tests/test_helpers/docker_helpers.py b/metadata-ingestion/tests/test_helpers/docker_helpers.py index 6ac46c54ab8a2d..329664ad14b128 100644 --- a/metadata-ingestion/tests/test_helpers/docker_helpers.py +++ b/metadata-ingestion/tests/test_helpers/docker_helpers.py @@ -1,6 +1,6 @@ import contextlib import subprocess -from typing import Optional, Union +from typing import Callable, Optional, Union import pytest import pytest_docker.plugin @@ -27,16 +27,15 @@ def wait_for_port( hostname: str = None, timeout: float = 30.0, pause: float = 0.5, + checker: Optional[Callable[[], bool]] = None, ) -> None: - # import pdb - - # breakpoint() try: - # port = docker_services.port_for(container_name, container_port) docker_services.wait_until_responsive( timeout=timeout, pause=pause, - check=lambda: is_responsive(container_name, container_port, hostname), + check=checker + if checker + else lambda: is_responsive(container_name, container_port, hostname), ) finally: # use check=True to raise an error if command gave bad exit code diff --git a/metadata-ingestion/tests/test_helpers/mce_helpers.py b/metadata-ingestion/tests/test_helpers/mce_helpers.py index 9da9bb7e689f50..5ba9aedbb7502a 100644 --- a/metadata-ingestion/tests/test_helpers/mce_helpers.py +++ b/metadata-ingestion/tests/test_helpers/mce_helpers.py @@ -2,8 +2,9 @@ import logging import os import pprint +import re import shutil -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union import deepdiff @@ -91,11 +92,23 @@ def assert_mces_equal( exclude_regex_paths=ignore_paths, ignore_order=True, ) - if clean_diff != diff: - logger.warning( - f"MCE-s differ, clean MCE-s are fine\n{pprint.pformat(diff)}" - ) + if not clean_diff: + logger.debug(f"MCE-s differ, clean MCE-s are fine\n{pprint.pformat(diff)}") diff = clean_diff + if diff: + # do some additional processing to emit helpful messages + output_urns = _get_entity_urns(output) + golden_urns = _get_entity_urns(golden) + in_golden_but_not_in_output = golden_urns - output_urns + in_output_but_not_in_golden = output_urns - golden_urns + if in_golden_but_not_in_output: + logger.info( + f"Golden file has {len(in_golden_but_not_in_output)} more urns: {in_golden_but_not_in_output}" + ) + if in_output_but_not_in_golden: + logger.info( + f"Golden file has {len(in_output_but_not_in_golden)} more urns: {in_output_but_not_in_golden}" + ) assert ( not diff @@ -192,6 +205,33 @@ def _element_matches_pattern( return (True, re.search(pattern, str(element)) is not None) +def get_entity_urns(events_file: str) -> Set[str]: + events = load_json_file(events_file) + assert isinstance(events, list) + return _get_entity_urns(events) + + +def _get_entity_urns(events_list: List[Dict]) -> Set[str]: + entity_type = "dataset" + # mce urns + mce_urns = set( + [ + _get_element(x, _get_mce_urn_path_spec(entity_type)) + for x in events_list + if _get_filter(mce=True, entity_type=entity_type)(x) + ] + ) + mcp_urns = set( + [ + _get_element(x, _get_mcp_urn_path_spec()) + for x in events_list + if _get_filter(mcp=True, entity_type=entity_type)(x) + ] + ) + all_urns = mce_urns.union(mcp_urns) + return all_urns + + def assert_mcp_entity_urn( filter: str, entity_type: str, regex_pattern: str, file: str ) -> int: @@ -234,6 +274,7 @@ def _get_mcp_urn_path_spec() -> List[str]: def assert_mce_entity_urn( filter: str, entity_type: str, regex_pattern: str, file: str ) -> int: + """Assert that all mce entity urns must match the regex pattern passed in. Return the number of events matched""" test_output = load_json_file(file) if isinstance(test_output, list): @@ -257,7 +298,11 @@ def assert_mce_entity_urn( def assert_for_each_entity( - entity_type: str, aspect_name: str, aspect_field_matcher: Dict[str, Any], file: str + entity_type: str, + aspect_name: str, + aspect_field_matcher: Dict[str, Any], + file: str, + exception_urns: List[str] = [], ) -> int: """Assert that an aspect name with the desired fields exists for each entity urn""" test_output = load_json_file(file) @@ -302,7 +347,7 @@ def assert_for_each_entity( aspect_val, [f] ), f"urn: {urn} -> Field {f} must match value {aspect_field_matcher[f]}, found {_get_element(aspect_val, [f])}" success.append(urn) - else: + elif urn not in exception_urns: print(f"Adding {urn} to failures") failures.append(urn) @@ -362,3 +407,62 @@ def assert_entity_mcp_aspect( ), f"urn: {mcp.entityUrn} -> Field {f} must match value {aspect_field_matcher[f]}, found {_get_element(aspect_val, [f])}" matches = matches + 1 return matches + + +def assert_entity_urn_not_like(entity_type: str, regex_pattern: str, file: str) -> int: + """Assert that there are no entity urns that match the regex pattern passed in. Returns the total number of events in the file""" + + test_output = load_json_file(file) + assert isinstance(test_output, list) + # mce urns + mce_urns = set( + [ + _get_element(x, _get_mce_urn_path_spec(entity_type)) + for x in test_output + if _get_filter(mce=True, entity_type=entity_type)(x) + ] + ) + mcp_urns = set( + [ + _get_element(x, _get_mcp_urn_path_spec()) + for x in test_output + if _get_filter(mcp=True, entity_type=entity_type)(x) + ] + ) + all_urns = mce_urns.union(mcp_urns) + print(all_urns) + matched_urns = [u for u in all_urns if re.match(regex_pattern, u)] + if matched_urns: + raise AssertionError(f"urns found that match the deny list {matched_urns}") + return len(test_output) + + +def assert_entity_urn_like(entity_type: str, regex_pattern: str, file: str) -> int: + """Assert that there exist entity urns that match the regex pattern passed in. Returns the total number of events in the file""" + + test_output = load_json_file(file) + assert isinstance(test_output, list) + # mce urns + mce_urns = set( + [ + _get_element(x, _get_mce_urn_path_spec(entity_type)) + for x in test_output + if _get_filter(mce=True, entity_type=entity_type)(x) + ] + ) + mcp_urns = set( + [ + _get_element(x, _get_mcp_urn_path_spec()) + for x in test_output + if _get_filter(mcp=True, entity_type=entity_type)(x) + ] + ) + all_urns = mce_urns.union(mcp_urns) + print(all_urns) + matched_urns = [u for u in all_urns if re.match(regex_pattern, u)] + if matched_urns: + return len(matched_urns) + else: + raise AssertionError( + f"No urns found that match the pattern {regex_pattern}. Full list is {all_urns}" + ) diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py index 8e0b1ce747ca34..e40ff55e935fe9 100644 --- a/metadata-ingestion/tests/unit/serde/test_serde.py +++ b/metadata-ingestion/tests/unit/serde/test_serde.py @@ -12,7 +12,11 @@ from datahub.emitter import mce_builder from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.file import iterate_mce_file -from datahub.metadata.schema_classes import MetadataChangeEventClass +from datahub.metadata.schema_classes import ( + MetadataChangeEventClass, + OwnershipClass, + _Aspect, +) from datahub.metadata.schemas import getMetadataChangeEventSchema from tests.test_helpers import mce_helpers from tests.test_helpers.click_helpers import run_datahub_cmd @@ -21,6 +25,18 @@ FROZEN_TIME = "2021-07-22 18:54:06" +def test_codegen_aspect_name(): + assert issubclass(OwnershipClass, _Aspect) + + assert OwnershipClass.ASPECT_NAME == "ownership" + assert OwnershipClass.get_aspect_name() == "ownership" + + +def test_cannot_instantiated_codegen_aspect(): + with pytest.raises(TypeError, match="instantiate"): + _Aspect() + + @freeze_time(FROZEN_TIME) @pytest.mark.parametrize( "json_filename", diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py index 9bbdebc9866071..317c26286438cb 100644 --- a/metadata-ingestion/tests/unit/test_athena_source.py +++ b/metadata-ingestion/tests/unit/test_athena_source.py @@ -25,7 +25,7 @@ def test_athena_uri(): ) assert ( config.get_sql_alchemy_url() - == "awsathena+rest://@athena.us-west-1.amazonaws.com:443/?s3_staging_dir=s3%3A%2F%2Fsample-staging-dir%2F&work_group=test-workgroup" + == "awsathena+rest://@athena.us-west-1.amazonaws.com:443/?s3_staging_dir=s3%3A%2F%2Fsample-staging-dir%2F&work_group=test-workgroup&catalog_name=awsdatacatalog&duration_seconds=3600" ) diff --git a/metadata-ingestion/tests/unit/test_bigquery_source.py b/metadata-ingestion/tests/unit/test_bigquery_source.py index 9e98254b84086b..85626373e20bf9 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_source.py @@ -1,5 +1,6 @@ import json import os +from datetime import datetime import pytest @@ -10,7 +11,6 @@ def test_bigquery_uri(): - config = BigQueryConfig.parse_obj( { "project_id": "test-project", @@ -20,7 +20,6 @@ def test_bigquery_uri(): def test_bigquery_uri_with_credential(): - expected_credential_json = { "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -67,7 +66,6 @@ def test_bigquery_uri_with_credential(): def test_simple_upstream_table_generation(): - a: BigQueryTableRef = BigQueryTableRef( project="test-project", dataset="test-dataset", table="a" ) @@ -97,7 +95,6 @@ def test_error_on_missing_config(): def test_upstream_table_generation_with_temporary_table_without_temp_upstream(): - a: BigQueryTableRef = BigQueryTableRef( project="test-project", dataset="test-dataset", table="a" ) @@ -143,7 +140,6 @@ def test_upstream_table_generation_with_temporary_table_with_temp_upstream(): def test_upstream_table_generation_with_temporary_table_with_multiple_temp_upstream(): - a: BigQueryTableRef = BigQueryTableRef( project="test-project", dataset="test-dataset", table="a" ) @@ -173,3 +169,158 @@ def test_upstream_table_generation_with_temporary_table_with_multiple_temp_upstr } upstreams = source.get_upstream_tables(str(a), []) assert list(upstreams).sort() == [c, e].sort() + + +def test_bq_get_profile_candidate_query_all_params(): + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": 1, + "profile_table_size_limit": 5, + "profile_table_row_limit": 50000, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + threshold_time = datetime.fromtimestamp(1648876349) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "row_count<50000 and ROUND(size_bytes/POW(10,9),2)<5 and last_modified_time>=1648876349000 " + ) + query = source.generate_profile_candidate_query(threshold_time, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_no_day_limit(): + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": None, + "profile_table_size_limit": 5, + "profile_table_row_limit": 50000, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "row_count<50000 and ROUND(size_bytes/POW(10,9),2)<5 " + ) + query = source.generate_profile_candidate_query(None, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_no_size_limit(): + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": 1, + "profile_table_size_limit": None, + "profile_table_row_limit": 50000, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + threshold_time = datetime.fromtimestamp(1648876349) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "row_count<50000 and last_modified_time>=1648876349000 " + ) + query = source.generate_profile_candidate_query(threshold_time, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_no_row_limit(): + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": 1, + "profile_table_size_limit": 5, + "profile_table_row_limit": None, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + threshold_time = datetime.fromtimestamp(1648876349) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "ROUND(size_bytes/POW(10,9),2)<5 and last_modified_time>=1648876349000 " + ) + query = source.generate_profile_candidate_query(threshold_time, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_all_null(): + + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": None, + "profile_table_size_limit": None, + "profile_table_row_limit": None, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + expected_query = "" + query = source.generate_profile_candidate_query(None, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_only_row(): + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": None, + "profile_table_size_limit": None, + "profile_table_row_limit": 50000, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "row_count<50000 " + ) + query = source.generate_profile_candidate_query(None, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_only_days(): + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": 1, + "profile_table_size_limit": None, + "profile_table_row_limit": None, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + threshold_time = datetime.fromtimestamp(1648876349) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "last_modified_time>=1648876349000 " + ) + query = source.generate_profile_candidate_query(threshold_time, "dataset_foo") + assert query == expected_query + + +def test_bq_get_profile_candidate_query_only_size(): + + config = BigQueryConfig.parse_obj( + { + "profiling": { + "profile_if_updated_since_days": None, + "profile_table_size_limit": 5, + "profile_table_row_limit": None, + } + } + ) + source = BigQuerySource(config=config, ctx=PipelineContext(run_id="test")) + expected_query = ( + "SELECT table_id, size_bytes, last_modified_time, row_count, FROM dataset_foo.__TABLES__ WHERE " + "ROUND(size_bytes/POW(10,9),2)<5 " + ) + query = source.generate_profile_candidate_query(None, "dataset_foo") + assert query == expected_query diff --git a/metadata-ingestion/tests/unit/test_bigquery_usage_source.py b/metadata-ingestion/tests/unit/test_bigquery_usage_source.py index 95c18a3839f574..94e0cc2475266f 100644 --- a/metadata-ingestion/tests/unit/test_bigquery_usage_source.py +++ b/metadata-ingestion/tests/unit/test_bigquery_usage_source.py @@ -1,7 +1,20 @@ import json import os -from datahub.ingestion.source.usage.bigquery_usage import BigQueryUsageConfig +from freezegun import freeze_time + +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.usage.bigquery_usage import ( + BQ_AUDIT_V1, + BigQueryTableRef, + BigQueryUsageConfig, + BigQueryUsageSource, +) +from datahub.ingestion.source_config.bigquery import ( + _BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX, +) + +FROZEN_TIME = "2021-07-20 00:00:00" def test_bigquery_uri_with_credential(): @@ -47,3 +60,143 @@ def test_bigquery_uri_with_credential(): if config._credentials_path: os.unlink(str(config._credentials_path)) raise e + + +@freeze_time(FROZEN_TIME) +def test_bigquery_filters_with_allow_filter(): + config = { + "project_id": "test-project", + "credential": { + "project_id": "test-project", + "private_key_id": "test-private-key", + "private_key": "random_private_key", + "client_email": "test@acryl.io", + "client_id": "test_client-id", + }, + "table_pattern": {"allow": ["test-regex", "test-regex-1"], "deny": []}, + } + expected_filter: str = """protoPayload.serviceName="bigquery.googleapis.com" +AND +( + ( + protoPayload.methodName="jobservice.jobcompleted" + AND + protoPayload.serviceData.jobCompletedEvent.eventName="query_job_completed" + AND + protoPayload.serviceData.jobCompletedEvent.job.jobStatus.state="DONE" + AND + NOT protoPayload.serviceData.jobCompletedEvent.job.jobStatus.error.code:* + ) + OR + ( + protoPayload.metadata.tableDataRead:* + ) +) +AND ( + +protoPayload.serviceData.jobCompletedEvent.job.jobStatistics.referencedTables.tableId =~ "test-regex|test-regex-1" + + +AND +protoPayload.serviceData.jobCompletedEvent.job.jobStatistics.referencedTables.tableId !~ "__TABLES_SUMMARY__|INFORMATION_SCHEMA" + + OR + protoPayload.metadata.tableDataRead.reason = "JOB" +) +AND +timestamp >= "2021-07-18T23:45:00Z" +AND +timestamp < "2021-07-21T00:15:00Z\"""" # noqa: W293 + + source = BigQueryUsageSource.create(config, PipelineContext(run_id="bq-usage-test")) + + # source: BigQueryUsageSource = BigQueryUsageSource( + # config=config, ctx=PipelineContext(run_id="test") + # ) + filter: str = source._generate_filter(BQ_AUDIT_V1) + assert filter == expected_filter + + +@freeze_time(FROZEN_TIME) +def test_bigquery_filters_with_deny_filter(): + config = { + "project_id": "test-project", + "credential": { + "project_id": "test-project", + "private_key_id": "test-private-key", + "private_key": "random_private_key", + "client_email": "test@acryl.io", + "client_id": "test_client-id", + }, + "table_pattern": { + "allow": ["test-regex", "test-regex-1"], + "deny": ["excluded_table_regex", "excluded-regex-2"], + }, + } + expected_filter: str = """protoPayload.serviceName="bigquery.googleapis.com" +AND +( + ( + protoPayload.methodName="jobservice.jobcompleted" + AND + protoPayload.serviceData.jobCompletedEvent.eventName="query_job_completed" + AND + protoPayload.serviceData.jobCompletedEvent.job.jobStatus.state="DONE" + AND + NOT protoPayload.serviceData.jobCompletedEvent.job.jobStatus.error.code:* + ) + OR + ( + protoPayload.metadata.tableDataRead:* + ) +) +AND ( + +protoPayload.serviceData.jobCompletedEvent.job.jobStatistics.referencedTables.tableId =~ "test-regex|test-regex-1" + + +AND +protoPayload.serviceData.jobCompletedEvent.job.jobStatistics.referencedTables.tableId !~ "__TABLES_SUMMARY__|INFORMATION_SCHEMA|excluded_table_regex|excluded-regex-2" + + OR + protoPayload.metadata.tableDataRead.reason = "JOB" +) +AND +timestamp >= "2021-07-18T23:45:00Z" +AND +timestamp < "2021-07-21T00:15:00Z\"""" # noqa: W293 + source = BigQueryUsageSource.create(config, PipelineContext(run_id="bq-usage-test")) + filter: str = source._generate_filter(BQ_AUDIT_V1) + assert filter == expected_filter + + +def test_bigquery_ref_extra_removal(): + table_ref = BigQueryTableRef("project-1234", "dataset-4567", "foo_*") + new_table_ref = table_ref.remove_extras(_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX) + assert new_table_ref.table == "foo" + assert new_table_ref.project == table_ref.project + assert new_table_ref.dataset == table_ref.dataset + + table_ref = BigQueryTableRef("project-1234", "dataset-4567", "foo_2022") + new_table_ref = table_ref.remove_extras(_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX) + assert new_table_ref.table == "foo" + assert new_table_ref.project == table_ref.project + assert new_table_ref.dataset == table_ref.dataset + + table_ref = BigQueryTableRef("project-1234", "dataset-4567", "foo_20222110") + new_table_ref = table_ref.remove_extras(_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX) + assert new_table_ref.table == "foo" + assert new_table_ref.project == table_ref.project + assert new_table_ref.dataset == table_ref.dataset + + table_ref = BigQueryTableRef("project-1234", "dataset-4567", "foo") + new_table_ref = table_ref.remove_extras(_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX) + assert new_table_ref.table == "foo" + assert new_table_ref.project == table_ref.project + assert new_table_ref.dataset == table_ref.dataset + + table_ref = BigQueryTableRef("project-1234", "dataset-4567", "foo_2016*") + new_table_ref = table_ref.remove_extras(_BIGQUERY_DEFAULT_SHARDED_TABLE_REGEX) + assert new_table_ref.table == "foo" + assert new_table_ref.project == table_ref.project + assert new_table_ref.dataset == table_ref.dataset diff --git a/metadata-ingestion/tests/unit/test_capability_report.py b/metadata-ingestion/tests/unit/test_capability_report.py new file mode 100644 index 00000000000000..08ada0386b0eae --- /dev/null +++ b/metadata-ingestion/tests/unit/test_capability_report.py @@ -0,0 +1,50 @@ +import json +from typing import cast + +from datahub.ingestion.api.source import ( + CapabilityReport, + SourceCapability, + TestConnectionReport, +) + + +def test_basic_capability_report(): + report = TestConnectionReport( + basic_connectivity=CapabilityReport( + capable=True, failure_reason=None, mitigation_message=None + ), + capability_report={ + "CONTAINERS": CapabilityReport( + capable=True, failure_reason=None, mitigation_message=None + ), + "SCHEMA_METADATA": CapabilityReport( + capable=True, failure_reason=None, mitigation_message=None + ), + "DESCRIPTIONS": CapabilityReport( + capable=False, + failure_reason="failed to get descriptions", + mitigation_message="Enable admin privileges for this account.", + ), + "DATA_PROFILING": CapabilityReport( + capable=True, failure_reason=None, mitigation_message=None + ), + SourceCapability.DOMAINS: CapabilityReport(capable=True), + }, + ) + print(report.as_obj()) + foo = cast(dict, report.as_obj()) + assert isinstance(foo, dict) + assert foo["capability_report"]["CONTAINERS"]["capable"] is True + assert foo["capability_report"]["SCHEMA_METADATA"]["capable"] is True + assert foo["capability_report"]["DESCRIPTIONS"]["capable"] is False + assert ( + foo["capability_report"]["DESCRIPTIONS"]["failure_reason"] + == "failed to get descriptions" + ) + assert ( + foo["capability_report"]["DESCRIPTIONS"]["mitigation_message"] + == "Enable admin privileges for this account." + ) + assert foo["capability_report"]["DOMAINS"]["capable"] is True + + assert isinstance(json.loads(report.as_json()), dict) diff --git a/metadata-ingestion/tests/unit/test_cli_utils.py b/metadata-ingestion/tests/unit/test_cli_utils.py index bead10f1ac8201..cb0b7c734ee0ac 100644 --- a/metadata-ingestion/tests/unit/test_cli_utils.py +++ b/metadata-ingestion/tests/unit/test_cli_utils.py @@ -1,3 +1,6 @@ +import os +from unittest import mock + from datahub.cli import cli_utils @@ -9,3 +12,50 @@ def test_first_non_null(): assert cli_utils.first_non_null(["3", "1", "2"]) == "3" assert cli_utils.first_non_null(["", "1", "2"]) == "1" assert cli_utils.first_non_null([" ", "1", "2"]) == "1" + + +@mock.patch.dict(os.environ, {"DATAHUB_GMS_HOST": "http://localhost:9092"}) +def test_correct_url_when_gms_host_in_old_format(): + assert cli_utils.get_details_from_env() == ("http://localhost:9092", None) + + +@mock.patch.dict( + os.environ, {"DATAHUB_GMS_HOST": "localhost", "DATAHUB_GMS_PORT": "8080"} +) +def test_correct_url_when_gms_host_and_port_set(): + assert cli_utils.get_details_from_env() == ("http://localhost:8080", None) + + +@mock.patch.dict( + os.environ, + { + "DATAHUB_GMS_URL": "https://example.com", + "DATAHUB_GMS_HOST": "localhost", + "DATAHUB_GMS_PORT": "8080", + }, +) +def test_correct_url_when_gms_host_port_url_set(): + assert cli_utils.get_details_from_env() == ("http://localhost:8080", None) + + +@mock.patch.dict( + os.environ, + { + "DATAHUB_GMS_URL": "https://example.com", + "DATAHUB_GMS_HOST": "localhost", + "DATAHUB_GMS_PORT": "8080", + "DATAHUB_GMS_PROTOCOL": "https", + }, +) +def test_correct_url_when_gms_host_port_url_protocol_set(): + assert cli_utils.get_details_from_env() == ("https://localhost:8080", None) + + +@mock.patch.dict( + os.environ, + { + "DATAHUB_GMS_URL": "https://example.com", + }, +) +def test_correct_url_when_url_set(): + assert cli_utils.get_details_from_env() == ("https://example.com", None) diff --git a/metadata-ingestion/tests/unit/test_csv_enricher_source.py b/metadata-ingestion/tests/unit/test_csv_enricher_source.py new file mode 100644 index 00000000000000..efbbbf1fa4d079 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_csv_enricher_source.py @@ -0,0 +1,206 @@ +from typing import Dict, List, Union +from unittest import mock + +from datahub.emitter import mce_builder +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.csv_enricher import CSVEnricherConfig, CSVEnricherSource +from datahub.metadata.schema_classes import ( + GlossaryTermAssociationClass, + OwnerClass, + OwnershipSourceClass, + OwnershipTypeClass, + TagAssociationClass, +) + +DATASET_URN = ( + "urn:li:dataset:(urn:li:dataPlatform:bigquery,test_dataset.test.Test,PROD)" +) +DATASET_ENTITY_TYPE = "dataset" + + +def create_owners_list_from_urn_list( + owner_urns: List[str], source_type: str +) -> List[OwnerClass]: + ownership_source_type: Union[None, OwnershipSourceClass] = None + if source_type: + ownership_source_type = OwnershipSourceClass(type=source_type) + owners_list = [ + OwnerClass( + owner=owner_urn, + type=OwnershipTypeClass.DATAOWNER, + source=ownership_source_type, + ) + for owner_urn in owner_urns + ] + return owners_list + + +def create_mocked_csv_enricher_source() -> CSVEnricherSource: + ctx = PipelineContext("test-run-id") + graph = mock.MagicMock() + graph.get_ownership.return_value = mce_builder.make_ownership_aspect_from_urn_list( + ["urn:li:corpuser:olduser1"], "AUDIT" + ) + graph.get_glossary_terms.return_value = ( + mce_builder.make_glossary_terms_aspect_from_urn_list( + ["urn:li:glossaryTerm:oldterm1", "urn:li:glossaryTerm:oldterm2"] + ) + ) + graph.get_tags.return_value = mce_builder.make_global_tag_aspect_with_tag_list( + ["oldtag1", "oldtag2"] + ) + graph.get_aspect_v2.return_value = None + graph.get_domain.return_value = None + ctx.graph = graph + return CSVEnricherSource( + CSVEnricherConfig(**create_base_csv_enricher_config()), ctx + ) + + +def create_base_csv_enricher_config() -> Dict: + return dict( + { + "filename": "../integration/csv_enricher/csv_enricher_test_data.csv", + "write_semantics": "PATCH", + "delimiter": ",", + "array_delimiter": "|", + }, + ) + + +def test_get_resource_glossary_terms_work_unit_no_terms(): + source = create_mocked_csv_enricher_source() + maybe_terms_wu = source.get_resource_glossary_terms_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, [] + ) + assert not maybe_terms_wu + + +def test_get_resource_glossary_terms_no_new_glossary_terms(): + source = create_mocked_csv_enricher_source() + new_glossary_terms = [ + "urn:li:glossaryTerm:oldterm1", + "urn:li:glossaryTerm:oldterm2", + ] + term_associations: List[GlossaryTermAssociationClass] = [ + GlossaryTermAssociationClass(term) for term in new_glossary_terms + ] + maybe_terms_wu = source.get_resource_glossary_terms_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, term_associations + ) + assert not maybe_terms_wu + + +def test_get_resource_glossary_terms_work_unit_produced(): + source = create_mocked_csv_enricher_source() + new_glossary_terms = [ + "urn:li:glossaryTerm:newterm1", + "urn:li:glossaryTerm:newterm2", + ] + term_associations: List[GlossaryTermAssociationClass] = [ + GlossaryTermAssociationClass(term) for term in new_glossary_terms + ] + maybe_terms_wu = source.get_resource_glossary_terms_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, term_associations + ) + assert maybe_terms_wu + + +def test_get_resource_tags_work_unit_no_tags(): + source = create_mocked_csv_enricher_source() + maybe_tags_wu = source.get_resource_tags_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, [] + ) + assert not maybe_tags_wu + + +def test_get_resource_tags_no_new_tags(): + source = create_mocked_csv_enricher_source() + new_tags = ["urn:li:tag:oldtag1", "urn:li:tag:oldtag2"] + tag_associations: List[TagAssociationClass] = [ + TagAssociationClass(tag) for tag in new_tags + ] + maybe_tags_wu = source.get_resource_tags_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, tag_associations + ) + assert not maybe_tags_wu + + +def test_get_resource_tags_work_unit_produced(): + source = create_mocked_csv_enricher_source() + new_tags = ["urn:li:tag:newtag1", "urn:li:tag:newtag2"] + tag_associations: List[TagAssociationClass] = [ + TagAssociationClass(tag) for tag in new_tags + ] + maybe_tags_wu = source.get_resource_tags_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, tag_associations + ) + assert maybe_tags_wu + + +def test_get_resource_owners_work_unit_no_terms(): + source = create_mocked_csv_enricher_source() + maybe_owners_wu = source.get_resource_owners_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, [] + ) + assert not maybe_owners_wu + + +def test_get_resource_owners_no_new_owners(): + source = create_mocked_csv_enricher_source() + new_owners = ["urn:li:corpuser:owner1", "urn:li:corpuser:owner2"] + owners: List[OwnerClass] = [ + OwnerClass(owner, type=OwnershipTypeClass.NONE) for owner in new_owners + ] + maybe_owners_wu = source.get_resource_owners_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, owners + ) + assert maybe_owners_wu + + +def test_get_resource_owners_work_unit_produced(): + source = create_mocked_csv_enricher_source() + new_owners = ["urn:li:corpuser:owner1", "urn:li:corpuser:owner2"] + owners: List[OwnerClass] = [ + OwnerClass(owner, type=OwnershipTypeClass.NONE) for owner in new_owners + ] + maybe_owners_wu = source.get_resource_owners_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, owners + ) + assert maybe_owners_wu + + +def test_get_resource_description_no_description(): + source = create_mocked_csv_enricher_source() + new_description = None + maybe_description_wu = source.get_resource_description_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, new_description + ) + assert not maybe_description_wu + + +def test_get_resource_description_work_unit_produced(): + source = create_mocked_csv_enricher_source() + new_description = "description" + maybe_description_wu = source.get_resource_description_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, new_description + ) + assert maybe_description_wu + + +def test_get_resource_domain_no_domain(): + source = create_mocked_csv_enricher_source() + new_domain = None + maybe_domain_wu = source.get_resource_domain_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, new_domain + ) + assert not maybe_domain_wu + + +def test_get_resource_domain_work_unit_produced(): + source = create_mocked_csv_enricher_source() + new_domain = "domain" + maybe_domain_wu = source.get_resource_domain_work_unit( + DATASET_URN, DATASET_ENTITY_TYPE, new_domain + ) + assert maybe_domain_wu diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py index edf6bc8a14549d..e6b3aa4fa0f066 100644 --- a/metadata-ingestion/tests/unit/test_dbt_source.py +++ b/metadata-ingestion/tests/unit/test_dbt_source.py @@ -1,6 +1,8 @@ from typing import Dict, List, Union from unittest import mock +from pydantic import ValidationError + from datahub.emitter import mce_builder from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.dbt import DBTConfig, DBTSource @@ -168,3 +170,86 @@ def test_dbt_source_patching_terms(): assert len(transformed_terms) == 3 for transformed_term in transformed_terms: assert transformed_term.urn in expected_terms + + +def test_dbt_entity_emission_configuration(): + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "entities_enabled": {"models": "Only", "seeds": "Only"}, + } + try: + DBTConfig.parse_obj(config_dict) + except ValidationError as ve: + assert len(ve.errors()) == 1 + assert ( + "Cannot have more than 1 type of entity emission set to ONLY" + in ve.errors()[0]["msg"] + ) + # valid config + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "entities_enabled": {"models": "Yes", "seeds": "Only"}, + } + DBTConfig.parse_obj(config_dict) + + +def test_dbt_entity_emission_configuration_helpers(): + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "entities_enabled": { + "models": "Only", + }, + } + config = DBTConfig.parse_obj(config_dict) + assert config.entities_enabled.can_emit_node_type("model") + assert not config.entities_enabled.can_emit_node_type("source") + assert not config.entities_enabled.can_emit_node_type("test") + assert not config.entities_enabled.can_emit_test_results + + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + } + config = DBTConfig.parse_obj(config_dict) + assert config.entities_enabled.can_emit_node_type("model") + assert config.entities_enabled.can_emit_node_type("source") + assert config.entities_enabled.can_emit_node_type("test") + assert config.entities_enabled.can_emit_test_results + + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "entities_enabled": { + "test_results": "Only", + }, + } + config = DBTConfig.parse_obj(config_dict) + assert not config.entities_enabled.can_emit_node_type("model") + assert not config.entities_enabled.can_emit_node_type("source") + assert not config.entities_enabled.can_emit_node_type("test") + assert config.entities_enabled.can_emit_test_results + + config_dict = { + "manifest_path": "dummy_path", + "catalog_path": "dummy_path", + "target_platform": "dummy_platform", + "entities_enabled": { + "test_results": "Yes", + "test_definitions": "Yes", + "models": "No", + "sources": "No", + }, + } + config = DBTConfig.parse_obj(config_dict) + assert not config.entities_enabled.can_emit_node_type("model") + assert not config.entities_enabled.can_emit_node_type("source") + assert config.entities_enabled.can_emit_node_type("test") + assert config.entities_enabled.can_emit_test_results diff --git a/metadata-ingestion/tests/unit/test_iceberg.py b/metadata-ingestion/tests/unit/test_iceberg.py index 949f6d10cc0577..a6aeb1919c6082 100644 --- a/metadata-ingestion/tests/unit/test_iceberg.py +++ b/metadata-ingestion/tests/unit/test_iceberg.py @@ -4,7 +4,7 @@ if sys.version_info < (3, 7): pytest.skip("iceberg not available for python < 3.7", allow_module_level=True) -from typing import Any +from typing import Any, Optional from iceberg.api import types as IcebergTypes from iceberg.api.types.types import NestedField @@ -13,18 +13,13 @@ from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.azure.azure_common import AdlsSourceConfig from datahub.ingestion.source.iceberg.iceberg import IcebergSource, IcebergSourceConfig -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( - ArrayType, - MapType, - SchemaField, -) +from datahub.metadata.com.linkedin.pegasus2avro.schema import ArrayType, SchemaField from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, BytesTypeClass, DateTypeClass, FixedTypeClass, - MapTypeClass, NumberTypeClass, RecordTypeClass, StringTypeClass, @@ -44,7 +39,7 @@ def with_iceberg_source() -> IcebergSource: def assert_field( schema_field: SchemaField, - expected_description: str, + expected_description: Optional[str], expected_nullable: bool, expected_type: Any, ) -> None: @@ -267,64 +262,62 @@ def test_iceberg_list_to_schema_field( @pytest.mark.parametrize( - "iceberg_type, expected_map_value_type", + "iceberg_type, expected_map_type", [ - (IcebergTypes.BinaryType.get(), "bytes"), - (IcebergTypes.BooleanType.get(), "boolean"), - (IcebergTypes.DateType.get(), "date"), + (IcebergTypes.BinaryType.get(), BytesTypeClass), + (IcebergTypes.BooleanType.get(), BooleanTypeClass), + (IcebergTypes.DateType.get(), DateTypeClass), ( IcebergTypes.DecimalType.of(3, 2), - "decimal", + NumberTypeClass, ), - (IcebergTypes.DoubleType.get(), "double"), - (IcebergTypes.FixedType.of_length(4), "fixed"), - (IcebergTypes.FloatType.get(), "float"), - (IcebergTypes.IntegerType.get(), "int"), - (IcebergTypes.LongType.get(), "long"), - (IcebergTypes.StringType.get(), "string"), + (IcebergTypes.DoubleType.get(), NumberTypeClass), + (IcebergTypes.FixedType.of_length(4), FixedTypeClass), + (IcebergTypes.FloatType.get(), NumberTypeClass), + (IcebergTypes.IntegerType.get(), NumberTypeClass), + (IcebergTypes.LongType.get(), NumberTypeClass), + (IcebergTypes.StringType.get(), StringTypeClass), ( IcebergTypes.TimestampType.with_timezone(), - "timestamp-micros", + TimeTypeClass, ), ( IcebergTypes.TimestampType.without_timezone(), - "timestamp-micros", + TimeTypeClass, ), - (IcebergTypes.TimeType.get(), "time-micros"), + (IcebergTypes.TimeType.get(), TimeTypeClass), ( IcebergTypes.UUIDType.get(), - "uuid", + StringTypeClass, ), ], ) def test_iceberg_map_to_schema_field( - iceberg_type: IcebergTypes.PrimitiveType, expected_map_value_type: Any + iceberg_type: IcebergTypes.PrimitiveType, expected_map_type: Any ) -> None: """ - Test converting a map typed Iceberg field to a MapType SchemaField, including the map value type. + Test converting a map typed Iceberg field to a MapType SchemaField, where the key is the same type as the value. """ map_column: NestedField = NestedField.required( 1, "mapField", - IcebergTypes.MapType.of_required( - 11, 12, IcebergTypes.StringType.get(), iceberg_type - ), + IcebergTypes.MapType.of_required(11, 12, iceberg_type, iceberg_type), "documentation", ) iceberg_source_instance = with_iceberg_source() schema_fields = iceberg_source_instance._get_schema_fields_for_column(map_column) - assert len(schema_fields) == 1, f"Expected 1 field, but got {len(schema_fields)}" - assert_field(schema_fields[0], map_column.doc, map_column.is_optional, MapTypeClass) - assert isinstance( - schema_fields[0].type.type, MapType - ), f"Field type {schema_fields[0].type.type} was expected to be {MapType}" - mapType: MapType = schema_fields[0].type.type - assert ( - mapType.keyType == "string" - ), f"Map key type {mapType.keyType} should always be a string" - assert ( - mapType.valueType == expected_map_value_type - ), f"Map value type {mapType.valueType} was expected to be {expected_map_value_type}" + # Converting an Iceberg Map type will be done by creating an array of struct(key, value) records. + # The first field will be the array. + assert len(schema_fields) == 3, f"Expected 3 fields, but got {len(schema_fields)}" + assert_field( + schema_fields[0], map_column.doc, map_column.is_optional, ArrayTypeClass + ) + + # The second field will be the key type + assert_field(schema_fields[1], None, False, expected_map_type) + + # The third field will be the value type + assert_field(schema_fields[2], None, True, expected_map_type) @pytest.mark.parametrize( diff --git a/metadata-ingestion/tests/unit/test_mapping.py b/metadata-ingestion/tests/unit/test_mapping.py index 5ed29be590cb1c..aea1d8ddd9a548 100644 --- a/metadata-ingestion/tests/unit/test_mapping.py +++ b/metadata-ingestion/tests/unit/test_mapping.py @@ -2,10 +2,12 @@ from datahub.metadata.com.linkedin.pegasus2avro.common import GlobalTags from datahub.metadata.schema_classes import ( + GlobalTagsClass, GlossaryTermsClass, OwnerClass, OwnershipClass, OwnershipSourceTypeClass, + OwnershipTypeClass, ) from datahub.utilities.mapping import OperationProcessor @@ -144,3 +146,88 @@ def test_operation_processor_no_email_strip_source_type_not_null(): new_owner: OwnerClass = ownership_aspect.owners[0] assert new_owner.owner == "urn:li:corpuser:test_user@abc.com" assert new_owner.source and new_owner.source.type == "SERVICE" + + +def test_operation_processor_advanced_matching_owners(): + raw_props = { + "user_owner": "@test_user@abc.com", + } + processor = OperationProcessor( + operation_defs={ + "user_owner": { + "match": "^@(.*)", + "operation": "add_owner", + "config": {"owner_type": "group"}, + }, + }, + owner_source_type="SOURCE_CONTROL", + ) + aspect_map = processor.process(raw_props) + assert "add_owner" in aspect_map + + ownership_aspect: OwnershipClass = aspect_map["add_owner"] + assert len(ownership_aspect.owners) == 1 + new_owner: OwnerClass = ownership_aspect.owners[0] + assert new_owner.owner == "urn:li:corpGroup:test_user@abc.com" + assert new_owner.source and new_owner.source.type == "SOURCE_CONTROL" + + +def test_operation_processor_ownership_category(): + raw_props = {"user_owner": "@test_user", "business_owner": "alice"} + processor = OperationProcessor( + operation_defs={ + "user_owner": { + "match": "^@(.*)", + "operation": "add_owner", + "config": { + "owner_type": "group", + "owner_category": OwnershipTypeClass.DATA_STEWARD, + }, + }, + "business_owner": { + "match": ".*", + "operation": "add_owner", + "config": { + "owner_type": "user", + "owner_category": OwnershipTypeClass.BUSINESS_OWNER, + }, + }, + }, + owner_source_type="SOURCE_CONTROL", + ) + aspect_map = processor.process(raw_props) + assert "add_owner" in aspect_map + + ownership_aspect: OwnershipClass = aspect_map["add_owner"] + assert len(ownership_aspect.owners) == 2 + new_owner: OwnerClass = ownership_aspect.owners[0] + assert new_owner.owner == "urn:li:corpGroup:test_user" + assert new_owner.source and new_owner.source.type == "SOURCE_CONTROL" + assert new_owner.type and new_owner.type == OwnershipTypeClass.DATA_STEWARD + + new_owner = ownership_aspect.owners[1] + assert new_owner.owner == "urn:li:corpuser:alice" + assert new_owner.source and new_owner.source.type == "SOURCE_CONTROL" + assert new_owner.type and new_owner.type == OwnershipTypeClass.BUSINESS_OWNER + + +def test_operation_processor_advanced_matching_tags(): + raw_props = { + "case": "PLT-4567", + } + processor = OperationProcessor( + operation_defs={ + "case": { + "match": "^PLT-(.*)", + "operation": "add_tag", + "config": {"tag": "case_{{ $match }}"}, + }, + }, + owner_source_type="SOURCE_CONTROL", + ) + aspect_map = processor.process(raw_props) + assert "add_tag" in aspect_map + + tag_aspect: GlobalTagsClass = aspect_map["add_tag"] + assert len(tag_aspect.tags) == 1 + assert tag_aspect.tags[0].tag == "urn:li:tag:case_4567" diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/test_pipeline.py index 827adc8686ce61..25a9b057f01555 100644 --- a/metadata-ingestion/tests/unit/test_pipeline.py +++ b/metadata-ingestion/tests/unit/test_pipeline.py @@ -66,7 +66,7 @@ def test_configure_without_sink(self, mock_emitter, mock_graph): assert pipeline.config.sink.type == "datahub-rest" assert pipeline.config.sink.config == { "server": "http://localhost:8080", - "token": "", + "token": None, } @freeze_time(FROZEN_TIME) @@ -109,7 +109,7 @@ def test_configure_with_rest_sink_initializes_graph( @freeze_time(FROZEN_TIME) @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True) - def test_configure_with_file_sink_does_not_init_graph(self, mock_source): + def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_path): pipeline = Pipeline.create( { "source": { @@ -119,7 +119,7 @@ def test_configure_with_file_sink_does_not_init_graph(self, mock_source): "sink": { "type": "file", "config": { - "filename": "test.json", + "filename": str(tmp_path / "test.json"), }, }, } @@ -127,7 +127,7 @@ def test_configure_with_file_sink_does_not_init_graph(self, mock_source): # assert that the default sink config is for a DatahubRestSink assert isinstance(pipeline.config.sink, DynamicTypedConfig) assert pipeline.config.sink.type == "file" - assert pipeline.config.sink.config == {"filename": "test.json"} + assert pipeline.config.sink.config == {"filename": str(tmp_path / "test.json")} assert pipeline.ctx.graph is None, "DataHubGraph should not be initialized" @freeze_time(FROZEN_TIME) diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py index 95d368a9498dbf..70e6f816949c4f 100644 --- a/metadata-ingestion/tests/unit/test_redash_source.py +++ b/metadata-ingestion/tests/unit/test_redash_source.py @@ -487,6 +487,7 @@ def test_get_dashboard_snapshot(): "urn:li:chart:(redash,9)", "urn:li:chart:(redash,8)", ], + datasets=[], lastModified=ChangeAuditStamps( created=AuditStamp( time=1628882055288, actor="urn:li:corpuser:unknown" diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/test_snowflake_source.py index 3bbe321838c497..631cf2d9db67d8 100644 --- a/metadata-ingestion/tests/unit/test_snowflake_source.py +++ b/metadata-ingestion/tests/unit/test_snowflake_source.py @@ -1,7 +1,10 @@ +from unittest.mock import MagicMock, patch + import pytest from datahub.configuration.common import ConfigurationError, OauthConfiguration -from datahub.ingestion.source.sql.snowflake import SnowflakeConfig +from datahub.ingestion.api.source import SourceCapability +from datahub.ingestion.source.sql.snowflake import SnowflakeConfig, SnowflakeSource def test_snowflake_source_throws_error_on_account_id_missing(): @@ -162,3 +165,246 @@ def test_options_contain_connect_args(): ) connect_args = config.get_options().get("connect_args") assert connect_args is not None + + +@patch("snowflake.connector.connect") +def test_test_connection_failure(mock_connect): + mock_connect.side_effect = Exception("Failed to connect to snowflake") + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeSource.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert not report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason + assert "Failed to connect to snowflake" in report.basic_connectivity.failure_reason + + +@patch("snowflake.connector.connect") +def test_test_connection_basic_success(mock_connect): + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeSource.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + + +def setup_mock_connect(mock_connect, query_results=None): + def default_query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE")] + # Unreachable code + raise Exception() + + connection_mock = MagicMock() + cursor_mock = MagicMock() + cursor_mock.execute.side_effect = ( + query_results if query_results is not None else default_query_results + ) + connection_mock.cursor.return_value = cursor_mock + mock_connect.return_value = connection_mock + + +@patch("snowflake.connector.connect") +def test_test_connection_no_warehouse(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [(None,)] + elif query == 'show grants to role "TEST_ROLE"': + return [ + ("", "USAGE", "DATABASE", "DB1"), + ("", "USAGE", "SCHEMA", "DB1.SCHEMA1"), + ("", "REFERENCES", "TABLE", "DB1.SCHEMA1.TABLE1"), + ] + elif query == 'show grants to role "PUBLIC"': + return [] + # Unreachable code + raise Exception() + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + setup_mock_connect(mock_connect, query_results) + report = SnowflakeSource.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + + assert report.capability_report + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert not report.capability_report[SourceCapability.SCHEMA_METADATA].capable + failure_reason = report.capability_report[ + SourceCapability.SCHEMA_METADATA + ].failure_reason + assert failure_reason + + assert "Current role does not have permissions to use warehouse" in failure_reason + + +@patch("snowflake.connector.connect") +def test_test_connection_capability_schema_failure(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE",)] + elif query == 'show grants to role "TEST_ROLE"': + return [("", "USAGE", "DATABASE", "DB1")] + elif query == 'show grants to role "PUBLIC"': + return [] + # Unreachable code + raise Exception() + + setup_mock_connect(mock_connect, query_results) + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeSource.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + assert report.capability_report + + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert not report.capability_report[SourceCapability.SCHEMA_METADATA].capable + assert ( + report.capability_report[SourceCapability.SCHEMA_METADATA].failure_reason + is not None + ) + + +@patch("snowflake.connector.connect") +def test_test_connection_capability_schema_success(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE")] + elif query == 'show grants to role "TEST_ROLE"': + return [ + ["", "USAGE", "DATABASE", "DB1"], + ["", "USAGE", "SCHEMA", "DB1.SCHEMA1"], + ["", "REFERENCES", "TABLE", "DB1.SCHEMA1.TABLE1"], + ] + elif query == 'show grants to role "PUBLIC"': + return [] + # Unreachable code + raise Exception() + + setup_mock_connect(mock_connect, query_results) + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeSource.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + assert report.capability_report + + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert report.capability_report[SourceCapability.SCHEMA_METADATA].capable + assert report.capability_report[SourceCapability.DESCRIPTIONS].capable + assert not report.capability_report[SourceCapability.DATA_PROFILING].capable + assert ( + report.capability_report[SourceCapability.DATA_PROFILING].failure_reason + is not None + ) + assert not report.capability_report[SourceCapability.LINEAGE_COARSE].capable + assert ( + report.capability_report[SourceCapability.LINEAGE_COARSE].failure_reason + is not None + ) + + +@patch("snowflake.connector.connect") +def test_test_connection_capability_all_success(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE")] + elif query == 'show grants to role "TEST_ROLE"': + return [ + ("", "USAGE", "DATABASE", "DB1"), + ("", "USAGE", "SCHEMA", "DB1.SCHEMA1"), + ("", "SELECT", "TABLE", "DB1.SCHEMA1.TABLE1"), + ("", "USAGE", "ROLE", "TEST_USAGE_ROLE"), + ] + elif query == 'show grants to role "PUBLIC"': + return [] + elif query == 'show grants to role "TEST_USAGE_ROLE"': + return [ + ["", "USAGE", "DATABASE", "SNOWFLAKE"], + ["", "USAGE", "SCHEMA", "ACCOUNT_USAGE"], + ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY"], + ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY"], + ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES"], + ] + # Unreachable code + raise Exception() + + setup_mock_connect(mock_connect, query_results) + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeSource.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + assert report.capability_report + + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert report.capability_report[SourceCapability.SCHEMA_METADATA].capable + assert report.capability_report[SourceCapability.DATA_PROFILING].capable + assert report.capability_report[SourceCapability.DESCRIPTIONS].capable + assert report.capability_report[SourceCapability.LINEAGE_COARSE].capable diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index f30cd5ad3f16ba..c6686e4f68c7f0 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -1465,7 +1465,8 @@ def test_old_transformers_working_as_before(mock_time): dataset_mcps = [ make_generic_dataset_mcp(), make_generic_dataset_mcp( - aspect=DatasetPropertiesClass(description="Another test MCP") + aspect_name="datasetProperties", + aspect=DatasetPropertiesClass(description="Another test MCP"), ), EndOfStream(), ] diff --git a/metadata-ingestion/tox.ini b/metadata-ingestion/tox.ini deleted file mode 100644 index 060f7f5a3fdc63..00000000000000 --- a/metadata-ingestion/tox.ini +++ /dev/null @@ -1,45 +0,0 @@ -# tox (https://tox.readthedocs.io/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py3-quick,py3-full,py3-airflow1 - -[gh-actions] -python = - 3.6: py3-full, py3-airflow1 - 3.9: py3-full, py3-airflow1 - -# Providing optional features that add dependencies from setup.py as deps here -# allows tox to recreate testenv when new dependencies are added to setup.py. -# Previous approach of using the tox global setting extras is not recommended -# as extras is only called when the testenv is created for the first time! -# see more here -> https://github.com/tox-dev/tox/issues/1105#issuecomment-448596282 - -[testenv] -passenv = SPARK_VERSION -deps = - .[dev] -commands = - pytest --cov={envsitepackagesdir}/datahub --cov={envsitepackagesdir}/datahub_provider \ - py3-quick,py3-airflow1: -m 'not integration and not slow_integration' --junit-xml=junit.quick.xml \ - py3-full: --cov-fail-under 65 --junit-xml=junit.full.xml \ - --continue-on-collection-errors \ - -vv - -setenv = - AIRFLOW_HOME = /tmp/airflow/thisshouldnotexist-{envname} - -[testenv:py3-airflow1] -deps = - .[dev-airflow1] - -c tests/airflow1-constraints.txt - -setenv = - AIRFLOW1_TEST = true - -[testenv:py3-full] -deps = - .[dev] - .[integration-tests] diff --git a/metadata-integration/java/as-a-library.md b/metadata-integration/java/as-a-library.md index 911db1a0a5e395..e4457b1bae4bc0 100644 --- a/metadata-integration/java/as-a-library.md +++ b/metadata-integration/java/as-a-library.md @@ -14,7 +14,7 @@ Follow the specific instructions for your build system to declare a dependency o ### Gradle Add the following to your build.gradle. ```gradle -implementation 'io.acryl:datahub-client:0.0.1' +implementation 'io.acryl:datahub-client:__version__' ``` ### Maven Add the following to your `pom.xml`. @@ -23,8 +23,8 @@ Add the following to your `pom.xml`. io.acryl datahub-client - - 0.0.1 + + __version__ ``` @@ -95,14 +95,127 @@ emitter.emit(mcpw, new Callback() { }); ``` -### Emitter Code +### REST Emitter Code If you're interested in looking at the REST emitter code, it is available [here](./datahub-client/src/main/java/datahub/client/rest/RestEmitter.java). ## Kafka Emitter -The Java package doesn't currently support a Kafka emitter, but this will be available shortly. +The Kafka emitter is a thin wrapper on top of the SerializingProducer class from `confluent-kafka` and offers a non-blocking interface for sending metadata events to DataHub. Use this when you want to decouple your metadata producer from the uptime of your datahub metadata server by utilizing Kafka as a highly available message bus. For example, if your DataHub metadata service is down due to planned or unplanned outages, you can still continue to collect metadata from your mission critical systems by sending it to Kafka. Also use this emitter when throughput of metadata emission is more important than acknowledgement of metadata being persisted to DataHub's backend store. +**_Note_**: The Kafka emitter uses Avro to serialize the Metadata events to Kafka. Changing the serializer will result in unprocessable events as DataHub currently expects the metadata events over Kafka to be serialized in Avro. + + +### Usage + +```java + + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import com.linkedin.dataset.DatasetProperties; +import datahub.client.kafka.KafkaEmitter; +import datahub.client.kafka.KafkaEmitterConfig; +import datahub.event.MetadataChangeProposalWrapper; + +// ... followed by + +// Creates the emitter with the default coordinates and settings +KafkaEmitterConfig.KafkaEmitterConfigBuilder builder = KafkaEmitterConfig.builder(); KafkaEmitterConfig config = builder.build(); +KafkaEmitter emitter = new KafkaEmitter(config); + +//Test if topic is available + +if(emitter.testConnection()){ + + MetadataChangeProposalWrapper mcpw = MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.user-table,PROD)") + .upsert() + .aspect(new DatasetProperties().setDescription("This is the canonical User profile dataset")) + .build(); + + // Blocking call using future + Future requestFuture = emitter.emit(mcpw, null).get(); + + // Non-blocking using callback + emitter.emit(mcpw, new Callback() { + + @Override + public void onFailure(Throwable exception) { + System.out.println("Failed to send with: " + exception); + } + @Override + public void onCompletion(MetadataWriteResponse metadataWriteResponse) { + if (metadataWriteResponse.isSuccess()) { + RecordMetadata metadata = (RecordMetadata) metadataWriteResponse.getUnderlyingResponse(); + System.out.println("Sent successfully over topic: " + metadata.topic()); + } else { + System.out.println("Failed to send with: " + metadataWriteResponse.getUnderlyingResponse()); + } + } + }); + +} +else { + System.out.println("Kafka service is down."); +} +``` +### Kafka Emitter Code + +If you're interested in looking at the Kafka emitter code, it is available [here](./datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java). + +## File Emitter + +The File emitter writes metadata change proposal events (MCPs) into a JSON file that can be later handed off to the Python [File source](docs/generated/ingestion/sources/file.md) for ingestion. This works analogous to the [File sink](../../metadata-ingestion/sink_docs/file.md) in Python. This mechanism can be used when the system producing metadata events doesn't have direct connection to DataHub's REST server or Kafka brokers. The generated JSON file can be transferred later and then ingested into DataHub using the [File source](docs/generated/ingestion/sources/file.md). + +### Usage + +```java + + +import datahub.client.file.FileEmitter; +import datahub.client.file.FileEmitterConfig; +import datahub.event.MetadataChangeProposalWrapper; + +// ... followed by + + +// Define output file co-ordinates +String outputFile = "/my/path/output.json"; + +//Create File Emitter +FileEmitter emitter = new FileEmitter(FileEmitterConfig.builder().fileName(outputFile).build()); + +// A couple of sample metadata events +MetadataChangeProposalWrapper mcpwOne = MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.user-table,PROD)") + .upsert() + .aspect(new DatasetProperties().setDescription("This is the canonical User profile dataset")) + .build(); + +MetadataChangeProposalWrapper mcpwTwo = MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-dataset.fact-orders-table,PROD)") + .upsert() + .aspect(new DatasetProperties().setDescription("This is the canonical Fact table for orders")) + .build(); + +MetadataChangeProposalWrapper[] mcpws = { mcpwOne, mcpwTwo }; +for (MetadataChangeProposalWrapper mcpw : mcpws) { + emitter.emit(mcpw); +} +emitter.close(); // calling close() is important to ensure file gets closed cleanly + +``` +### File Emitter Code + +If you're interested in looking at the File emitter code, it is available [here](./datahub-client/src/main/java/datahub/client/file/FileEmitter.java). + +### Support for S3, GCS etc. + +The File emitter only supports writing to the local filesystem currently. If you're interested in adding support for S3, GCS etc., contributions are welcome! ## Other Languages diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index a6edfca073d79f..78ba4e3723ec20 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -15,21 +15,43 @@ jar.enabled = false // Since we only want to build shadow jars, disabling the re dependencies { implementation project(':metadata-models') + compile externalDependency.avro_1_7 + constraints { + implementation('commons-collections:commons-collections:3.2.2') { + because 'Vulnerability Issue' + } + } shadow externalDependency.httpAsyncClient // we want our clients to provide this implementation externalDependency.jacksonDataBind implementation externalDependency.javaxValidation implementation externalDependency.springContext implementation externalDependency.swaggerAnnotations + implementation(externalDependency.kafkaAvroSerializer) { + exclude group: "org.apache.avro" + } + compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok testCompile externalDependency.httpAsyncClient // needed as shadow excludes it testCompile externalDependency.mockito testCompile externalDependency.mockServer testCompile externalDependency.mockServerClient + testCompile externalDependency.testContainers swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.33' } +task copyAvroSchemas { + dependsOn(':metadata-events:mxe-schemas:renameNamespace') + copy { + from file('../../../metadata-events/mxe-schemas/src/renamed/avro/com/linkedin/mxe/MetadataChangeProposal.avsc') + into file('./src/main/resources') + } +} + +compileJava.dependsOn copyAvroSchemas + + jacocoTestReport { dependsOn test // tests are required to run before generating the report } @@ -44,7 +66,6 @@ if (project.hasProperty("releaseVersion")) { try { // apply this plugin in a try-catch block so that we can handle cases without .git directory apply plugin: "com.palantir.git-version" - println("In else section") def details = versionDetails() detailedVersionString = gitVersion() version = details.lastTag @@ -100,6 +121,9 @@ shadowJar { archiveClassifier = '' dependencies { exclude(dependency('org.apache.httpcomponents:httpasyncclient')) + exclude 'LICENSE' + exclude 'NOTICE' + exclude 'LICENSE.txt' } mergeServiceFiles() // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why @@ -121,6 +145,21 @@ shadowJar { relocate 'com.github.benmanes.caffeine', 'datahub.shaded.com.github.benmanes.caffeine' relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework' relocate 'com.google.errorprone', 'datahub.shaded.com.google.errorprone' + // Below jars added for kafka emitter only + relocate 'org.apache.avro', 'datahub.shaded.org.apache.avro' + relocate 'org.codehaus.jackson', 'datahub.shaded.org.codehaus.jackson' + relocate 'com.thoughtworks.paranamer', 'datahub.shaded.com.thoughtworks.paranamer' + relocate 'org.xerial.snappy', 'datahub.shaded.org.xerial.snappy' + relocate 'org.apache.kafka', 'datahub.shaded.org.apache.kafka' + relocate 'io.confluent', 'datahub.shaded.io.confluent' + relocate 'org.apache.zookeeper', 'datahub.shaded.org.apache.zookeeper' + relocate 'org.apache.yetus', 'datahub.shaded.org.apache.yetus' + relocate 'jline', 'datahub.shaded.jline' + relocate 'org.apache.jute', 'datahub.shaded.org.apache.jute' + relocate 'org.I0Itec.zkclient', 'datahub.shaded.org.I0Itec.zkclient' + relocate 'net.jpountz', 'datahub.shaded.net.jpountz' + relocate 'com.github.luben', 'datahub.shaded.com.github.luben' + finalizedBy checkShadowJar } diff --git a/metadata-integration/java/datahub-client/scripts/check_jar.sh b/metadata-integration/java/datahub-client/scripts/check_jar.sh index 1b412cf2e27257..3665f110658a65 100755 --- a/metadata-integration/java/datahub-client/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-client/scripts/check_jar.sh @@ -23,7 +23,16 @@ jar -tvf $jarFile |\ grep -v "xml-header-style.xml" |\ grep -v "license.header" |\ grep -v "module-info.class" |\ - grep -v "client.properties" + grep -v "com/google/" |\ + grep -v "org/codehaus/" |\ + grep -v "client.properties" |\ + grep -v "kafka" |\ + grep -v "win/" |\ + grep -v "include/" |\ + grep -v "linux/" |\ + grep -v "darwin" |\ + grep -v "MetadataChangeProposal.avsc" |\ + grep -v "aix" if [ $? -ne 0 ]; then echo "✅ No unexpected class paths found in ${jarFile}" diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitter.java new file mode 100644 index 00000000000000..8dac652790f971 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitter.java @@ -0,0 +1,198 @@ +package datahub.client.file; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.data.template.JacksonDataTemplateCodec; +import com.linkedin.mxe.MetadataChangeProposal; + +import datahub.client.Callback; +import datahub.client.Emitter; +import datahub.client.MetadataWriteResponse; +import datahub.event.EventFormatter; +import datahub.event.MetadataChangeProposalWrapper; +import datahub.event.UpsertAspectRequest; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FileEmitter implements Emitter { + + private final EventFormatter eventFormatter; + private final FileEmitterConfig config; + private final ObjectMapper objectMapper = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + private final JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory()); + + private final BufferedWriter writer; + private final Future cachedSuccessFuture; + private final AtomicBoolean closed; + private boolean wroteSomething; + private static final String INDENT_4 = " "; + + /** + * The default constructor + * + * @param config + */ + public FileEmitter(FileEmitterConfig config) { + + this.config = config; + this.eventFormatter = this.config.getEventFormatter(); + + DefaultPrettyPrinter pp = new DefaultPrettyPrinter() + .withObjectIndenter(new DefaultIndenter(FileEmitter.INDENT_4, DefaultIndenter.SYS_LF)) + .withArrayIndenter(new DefaultIndenter(FileEmitter.INDENT_4, DefaultIndenter.SYS_LF)); + this.dataTemplateCodec.setPrettyPrinter(pp); + + try { + FileWriter fw = new FileWriter(config.getFileName(), false); + this.writer = new BufferedWriter(fw); + this.writer.append("["); + this.writer.newLine(); + this.closed = new AtomicBoolean(false); + } catch (IOException e) { + throw new RuntimeException("Error while creating file", e); + } + this.wroteSomething = false; + log.debug("Emitter created successfully for " + this.config.getFileName()); + + this.cachedSuccessFuture = new Future() { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public MetadataWriteResponse get() throws InterruptedException, ExecutionException { + return MetadataWriteResponse.builder().success(true).responseContent("MCP witten to File").build(); + } + + @Override + public MetadataWriteResponse get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return this.get(); + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + }; + } + + @Override + public void close() throws IOException { + this.writer.newLine(); + this.writer.append("]"); + this.writer.close(); + this.closed.set(true); + log.debug("Emitter closed for {}", this.config.getFileName()); + } + + @Override + public Future emit(@SuppressWarnings("rawtypes") MetadataChangeProposalWrapper mcpw, + Callback callback) throws IOException { + return emit(this.eventFormatter.convert(mcpw), callback); + } + + @Override + public Future emit(MetadataChangeProposal mcp, Callback callback) throws IOException { + if (this.closed.get()) { + String errorMsg = "File Emitter is already closed."; + log.error(errorMsg); + Future response = createFailureFuture(errorMsg); + if (callback != null) { + callback.onFailure(new Exception(errorMsg)); + } + return response; + } + try { + String serializedMCP = this.dataTemplateCodec.mapToString(mcp.data()); + if (wroteSomething) { + this.writer.append(","); + this.writer.newLine(); + } + this.writer.append(serializedMCP); + wroteSomething = true; + log.debug("MCP written successfully: {}", serializedMCP); + Future response = this.cachedSuccessFuture; + if (callback != null) { + try { + callback.onCompletion(response.get()); + } catch (InterruptedException | ExecutionException e) { + log.warn("Callback could not be executed.", e); + } + } + return response; + } catch (Throwable t) { + Future response = createFailureFuture(t.getMessage()); + if (callback != null) { + try { + callback.onFailure(t); + } catch (Exception e) { + log.warn("Callback could not be executed.", e); + } + } + return response; + } + } + + @Override + public boolean testConnection() throws IOException, ExecutionException, InterruptedException { + throw new UnsupportedOperationException("testConnection not relevant for File Emitter"); + } + + @Override + public Future emit(List request, Callback callback) throws IOException { + throw new UnsupportedOperationException("UpsertAspectRequest not relevant for File Emitter"); + } + + private Future createFailureFuture(String message) { + return new Future() { + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public MetadataWriteResponse get() throws InterruptedException, ExecutionException { + return MetadataWriteResponse.builder().success(false).responseContent(message).build(); + } + + @Override + public MetadataWriteResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, + TimeoutException { + return this.get(); + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + }; + } + +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitterConfig.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitterConfig.java new file mode 100644 index 00000000000000..c89edef81ef5e9 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/file/FileEmitterConfig.java @@ -0,0 +1,16 @@ +package datahub.client.file; + +import datahub.event.EventFormatter; +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class FileEmitterConfig { + @Builder.Default + @lombok.NonNull + private final String fileName = null; + @Builder.Default + private final EventFormatter eventFormatter = new EventFormatter(EventFormatter.Format.PEGASUS_JSON); + +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java new file mode 100644 index 00000000000000..ee0d459aaa7d3b --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/AvroSerializer.java @@ -0,0 +1,49 @@ +package datahub.client.kafka; + +import java.io.IOException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import com.google.common.annotations.VisibleForTesting; +import com.linkedin.mxe.MetadataChangeProposal; + +import datahub.event.EventFormatter; +import datahub.event.MetadataChangeProposalWrapper; + +class AvroSerializer { + + private final Schema _recordSchema; + private final Schema _genericAspectSchema; + private final EventFormatter _eventFormatter; + + public AvroSerializer() throws IOException { + _recordSchema = new Schema.Parser() + .parse(this.getClass().getClassLoader().getResourceAsStream("MetadataChangeProposal.avsc")); + _genericAspectSchema = this._recordSchema.getField("aspect").schema().getTypes().get(1); + _eventFormatter = new EventFormatter(EventFormatter.Format.PEGASUS_JSON); + } + + @VisibleForTesting + Schema getRecordSchema() { + return _recordSchema; + } + + public GenericRecord serialize(@SuppressWarnings("rawtypes") MetadataChangeProposalWrapper mcpw) throws IOException { + return serialize(_eventFormatter.convert(mcpw)); + } + + public GenericRecord serialize(MetadataChangeProposal mcp) throws IOException { + GenericRecord genericRecord = new GenericData.Record(this._recordSchema); + genericRecord.put("entityUrn", mcp.getEntityUrn().toString()); + GenericRecord genericAspect = new GenericData.Record(this._genericAspectSchema); + genericAspect.put("contentType", "application/json"); + genericAspect.put("value", mcp.getAspect().getValue().asByteBuffer()); + genericRecord.put("aspect", genericAspect); + genericRecord.put("aspectName", mcp.getAspectName()); + genericRecord.put("entityType", mcp.getEntityType()); + genericRecord.put("changeType", mcp.getChangeType()); + return genericRecord; + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java new file mode 100644 index 00000000000000..6d852e30cac48b --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitter.java @@ -0,0 +1,161 @@ +package datahub.client.kafka; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.avro.generic.GenericRecord; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ListTopicsOptions; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +import com.linkedin.mxe.MetadataChangeProposal; + +import datahub.client.Callback; +import datahub.client.Emitter; +import datahub.client.MetadataWriteResponse; +import datahub.event.MetadataChangeProposalWrapper; +import datahub.event.UpsertAspectRequest; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class KafkaEmitter implements Emitter { + + public static final String DEFAULT_MCP_KAFKA_TOPIC = "MetadataChangeProposal_v1"; + + private final KafkaEmitterConfig config; + private final KafkaProducer producer; + private final Properties kafkaConfigProperties; + private AvroSerializer _avroSerializer; + private static final int ADMIN_CLIENT_TIMEOUT_MS = 5000; + + /** + * The default constructor + * + * @param config + * @throws IOException + */ + public KafkaEmitter(KafkaEmitterConfig config) throws IOException { + this.config = config; + kafkaConfigProperties = new Properties(); + kafkaConfigProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.config.getBootstrap()); + kafkaConfigProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + org.apache.kafka.common.serialization.StringSerializer.class); + kafkaConfigProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + io.confluent.kafka.serializers.KafkaAvroSerializer.class); + kafkaConfigProperties.put("schema.registry.url", this.config.getSchemaRegistryUrl()); + kafkaConfigProperties.putAll(config.getSchemaRegistryConfig()); + kafkaConfigProperties.putAll(config.getProducerConfig()); + producer = new KafkaProducer(kafkaConfigProperties); + _avroSerializer = new AvroSerializer(); + } + + @Override + public void close() throws IOException { + producer.close(); + + } + + @Override + public Future emit(@SuppressWarnings("rawtypes") MetadataChangeProposalWrapper mcpw, + Callback datahubCallback) throws IOException { + return emit(this.config.getEventFormatter().convert(mcpw), datahubCallback); + } + + @Override + public Future emit(MetadataChangeProposal mcp, Callback datahubCallback) throws IOException { + GenericRecord genricRecord = _avroSerializer.serialize(mcp); + ProducerRecord record = new ProducerRecord<>(KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC, + mcp.getEntityUrn().toString(), genricRecord); + org.apache.kafka.clients.producer.Callback callback = new org.apache.kafka.clients.producer.Callback() { + + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + MetadataWriteResponse response = mapResponse(metadata, exception); + datahubCallback.onCompletion(response); + } + }; + log.debug("Emit: topic: {} \n record: {}", KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC, record); + Future future = this.producer.send(record, callback); + return mapFuture(future); + } + + private Future mapFuture(Future future) { + return new Future() { + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return future.cancel(mayInterruptIfRunning); + } + + @Override + public MetadataWriteResponse get() throws InterruptedException, ExecutionException { + RecordMetadata recordMetadata = future.get(); + return mapResponse(recordMetadata, null); + } + + @Override + public MetadataWriteResponse get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + RecordMetadata recordMetadata = future.get(timeout, unit); + return mapResponse(recordMetadata, null); + } + + @Override + public boolean isCancelled() { + return future.isCancelled(); + } + + @Override + public boolean isDone() { + return future.isDone(); + } + }; + + } + + @Override + public boolean testConnection() throws IOException, ExecutionException, InterruptedException { + try (AdminClient client = AdminClient.create(this.kafkaConfigProperties)) { + log.info("Available topics:" + + client.listTopics(new ListTopicsOptions().timeoutMs(ADMIN_CLIENT_TIMEOUT_MS)).listings().get()); + } catch (ExecutionException ex) { + log.error("Kafka is not available, timed out after {} ms", ADMIN_CLIENT_TIMEOUT_MS); + return false; + } + return true; + } + + @Override + public Future emit(List request, Callback callback) throws IOException { + throw new UnsupportedOperationException("UpsertAspectRequest cannot be sent over Kafka"); + } + + private static MetadataWriteResponse mapResponse(RecordMetadata metadata, Exception exception) { + + MetadataWriteResponse.MetadataWriteResponseBuilder builder = MetadataWriteResponse.builder(); + + if (exception == null) { + builder.success(true); + builder.underlyingResponse(metadata); + builder.responseContent(metadata.toString()); + } else { + builder.success(false); + builder.underlyingResponse(exception); + builder.responseContent(exception.toString()); + } + return builder.build(); + } + + public Properties getKafkaConfgiProperties() { + return kafkaConfigProperties; + } + +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitterConfig.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitterConfig.java new file mode 100644 index 00000000000000..9452dd5686ac79 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/kafka/KafkaEmitterConfig.java @@ -0,0 +1,55 @@ +package datahub.client.kafka; + +import java.io.InputStream; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.function.Consumer; + +import datahub.event.EventFormatter; +import lombok.Builder; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; + +@Value +@Builder +@Slf4j +public class KafkaEmitterConfig { + + public static final String CLIENT_VERSION_PROPERTY = "clientVersion"; + + @Builder.Default + private final String bootstrap = "localhost:9092"; + @Builder.Default + private final String schemaRegistryUrl = "http://localhost:8081"; + + @Builder.Default + private final Map schemaRegistryConfig = Collections.emptyMap(); + @Builder.Default + private final Map producerConfig = Collections.emptyMap(); + + @Builder.Default + private final EventFormatter eventFormatter = new EventFormatter(EventFormatter.Format.PEGASUS_JSON); + + public static class KafkaEmitterConfigBuilder { + + @SuppressWarnings("unused") + private String getVersion() { + try (InputStream foo = this.getClass().getClassLoader().getResourceAsStream("client.properties")) { + Properties properties = new Properties(); + properties.load(foo); + return properties.getProperty(CLIENT_VERSION_PROPERTY, "unknown"); + } catch (Exception e) { + log.warn("Unable to find a version for datahub-client. Will set to unknown", e); + return "unknown"; + } + } + + public KafkaEmitterConfigBuilder with(Consumer builderFunction) { + builderFunction.accept(this); + return this; + } + + } + +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java index 9193dd7333872a..ea825e6bbc54bb 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java @@ -1,5 +1,6 @@ package datahub.client.rest; +import com.google.common.annotations.VisibleForTesting; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -37,6 +38,15 @@ import datahub.event.UpsertAspectRequest; import lombok.extern.slf4j.Slf4j; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.nio.client.HttpAsyncClient; +import org.apache.http.ssl.SSLContextBuilder; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; + @ThreadSafe @Slf4j @@ -86,6 +96,17 @@ public RestEmitter(RestEmitterConfig config) { .setSocketTimeout(config.getTimeoutSec() * 1000) .build()); } + if (config.isDisableSslVerification()) { + HttpAsyncClientBuilder httpClientBuilder = this.config.getAsyncHttpClientBuilder(); + try { + httpClientBuilder + .setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { + throw new RuntimeException("Error while creating insecure http client", e); + } + } + this.httpClient = this.config.getAsyncHttpClientBuilder().build(); this.httpClient.start(); this.ingestProposalUrl = this.config.getServer() + "/aspects?action=ingestProposal"; @@ -313,4 +334,10 @@ public void cancelled() { Future requestFuture = httpClient.execute(httpPost, httpCallback); return new MetadataResponseFuture(requestFuture, responseAtomicReference, responseLatch); } + + @VisibleForTesting + HttpAsyncClient getHttpClient() { + return this.httpClient; + } + } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java index 3a83647ae3081a..f615c3ccb3e4fa 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java @@ -28,7 +28,9 @@ public class RestEmitterConfig { private final String server = "http://localhost:8080"; private final Integer timeoutSec; - + @Builder.Default + private final boolean disableSslVerification = false; + @Builder.Default private final String token = DEFAULT_AUTH_TOKEN; diff --git a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc new file mode 100644 index 00000000000000..8a4e6028ea2682 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc @@ -0,0 +1,164 @@ +{ + "type" : "record", + "name" : "MetadataChangeProposal", + "namespace" : "com.linkedin.pegasus2avro.mxe", + "doc" : "Kafka event for proposing a metadata change for an entity. A corresponding MetadataChangeLog is emitted when the change is accepted and committed, otherwise a FailedMetadataChangeProposal will be emitted instead.", + "fields" : [ { + "name" : "auditHeader", + "type" : [ "null", { + "type" : "record", + "name" : "KafkaAuditHeader", + "namespace" : "com.linkedin.events", + "doc" : "This header records information about the context of an event as it is emitted into kafka and is intended to be used by the kafka audit application. For more information see go/kafkaauditheader", + "fields" : [ { + "name" : "time", + "type" : "long", + "doc" : "The time at which the event was emitted into kafka.", + "compliance" : [ { + "policy" : "EVENT_TIME" + } ] + }, { + "name" : "server", + "type" : "string", + "doc" : "The fully qualified name of the host from which the event is being emitted.", + "compliance" : "NONE" + }, { + "name" : "instance", + "type" : [ "null", "string" ], + "doc" : "The instance on the server from which the event is being emitted. e.g. i001", + "default" : null, + "compliance" : "NONE" + }, { + "name" : "appName", + "type" : "string", + "doc" : "The name of the application from which the event is being emitted. see go/appname", + "compliance" : "NONE" + }, { + "name" : "messageId", + "type" : { + "type" : "fixed", + "name" : "UUID", + "size" : 16 + }, + "doc" : "A unique identifier for the message", + "compliance" : "NONE" + }, { + "name" : "auditVersion", + "type" : [ "null", "int" ], + "doc" : "The version that is being used for auditing. In version 0, the audit trail buckets events into 10 minute audit windows based on the EventHeader timestamp. In version 1, the audit trail buckets events as follows: if the schema has an outer KafkaAuditHeader, use the outer audit header timestamp for bucketing; else if the EventHeader has an inner KafkaAuditHeader use that inner audit header's timestamp for bucketing", + "default" : null, + "compliance" : "NONE" + }, { + "name" : "fabricUrn", + "type" : [ "null", "string" ], + "doc" : "The fabricUrn of the host from which the event is being emitted. Fabric Urn in the format of urn:li:fabric:{fabric_name}. See go/fabric.", + "default" : null, + "compliance" : "NONE" + }, { + "name" : "clusterConnectionString", + "type" : [ "null", "string" ], + "doc" : "This is a String that the client uses to establish some kind of connection with the Kafka cluster. The exact format of it depends on specific versions of clients and brokers. This information could potentially identify the fabric and cluster with which the client is producing to or consuming from.", + "default" : null, + "compliance" : "NONE" + } ] + } ], + "doc" : "Kafka audit header. Currently remains unused in the open source.", + "default" : null + }, { + "name" : "entityType", + "type" : "string", + "doc" : "Type of the entity being written to" + }, { + "name" : "entityUrn", + "type" : [ "null", "string" ], + "doc" : "Urn of the entity being written", + "default" : null, + "java" : { + "class" : "com.linkedin.pegasus2avro.common.urn.Urn" + } + }, { + "name" : "entityKeyAspect", + "type" : [ "null", { + "type" : "record", + "name" : "GenericAspect", + "doc" : "Generic record structure for serializing an Aspect", + "fields" : [ { + "name" : "value", + "type" : "bytes", + "doc" : "The value of the aspect, serialized as bytes." + }, { + "name" : "contentType", + "type" : "string", + "doc" : "The content type, which represents the fashion in which the aspect was serialized.\nThe only type currently supported is application/json." + } ] + } ], + "doc" : "Key aspect of the entity being written", + "default" : null + }, { + "name" : "changeType", + "type" : { + "type" : "enum", + "name" : "ChangeType", + "namespace" : "com.linkedin.pegasus2avro.events.metadata", + "doc" : "Descriptor for a change action", + "symbols" : [ "UPSERT", "CREATE", "UPDATE", "DELETE", "PATCH", "RESTATE" ], + "symbolDocs" : { + "CREATE" : "NOT SUPPORTED YET\ninsert if not exists. otherwise fail", + "DELETE" : "NOT SUPPORTED YET\ndelete action", + "PATCH" : "NOT SUPPORTED YET\npatch the changes instead of full replace", + "RESTATE" : "Restate an aspect, eg. in a index refresh.", + "UPDATE" : "NOT SUPPORTED YET\nupdate if exists. otherwise fail", + "UPSERT" : "insert if not exists. otherwise update" + } + }, + "doc" : "Type of change being proposed" + }, { + "name" : "aspectName", + "type" : [ "null", "string" ], + "doc" : "Aspect of the entity being written to\nNot filling this out implies that the writer wants to affect the entire entity\nNote: This is only valid for CREATE, UPSERT, and DELETE operations.", + "default" : null + }, { + "name" : "aspect", + "type" : [ "null", "GenericAspect" ], + "doc" : "The value of the new aspect.", + "default" : null + }, { + "name" : "systemMetadata", + "type" : [ "null", { + "type" : "record", + "name" : "SystemMetadata", + "doc" : "Metadata associated with each metadata change that is processed by the system", + "fields" : [ { + "name" : "lastObserved", + "type" : [ "long", "null" ], + "doc" : "The timestamp the metadata was observed at", + "default" : 0 + }, { + "name" : "runId", + "type" : [ "string", "null" ], + "doc" : "The run id that produced the metadata. Populated in case of batch-ingestion.", + "default" : "no-run-id-provided" + }, { + "name" : "registryName", + "type" : [ "null", "string" ], + "doc" : "The model registry name that was used to process this event", + "default" : null + }, { + "name" : "registryVersion", + "type" : [ "null", "string" ], + "doc" : "The model registry version that was used to process this event", + "default" : null + }, { + "name" : "properties", + "type" : [ "null", { + "type" : "map", + "values" : "string" + } ], + "doc" : "Additional properties", + "default" : null + } ] + } ], + "doc" : "A string->string map of custom properties that one might want to attach to an event", + "default" : null + } ] +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/file/FileEmitterTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/file/FileEmitterTest.java new file mode 100644 index 00000000000000..d5f556fa04991b --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/file/FileEmitterTest.java @@ -0,0 +1,136 @@ +package datahub.client.file; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.fasterxml.jackson.core.exc.StreamReadException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DatabindException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.JacksonDataTemplateCodec; +import com.linkedin.dataset.DatasetProperties; +import com.linkedin.mxe.MetadataChangeProposal; + +import datahub.client.Callback; +import datahub.client.MetadataWriteResponse; +import datahub.event.MetadataChangeProposalWrapper; + +public class FileEmitterTest { + private final ObjectMapper objectMapper = new ObjectMapper(); + private final JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(objectMapper.getFactory()); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testFileEmitter() throws IOException { + + InputStream goldenFileStream = ClassLoader.getSystemResourceAsStream("golden_files/mcps_golden.json"); + + String tempRoot = tempFolder.getRoot().toString(); + String outputFile = tempRoot + "/test.json"; + FileEmitter emitter = new FileEmitter(FileEmitterConfig.builder().fileName(outputFile).build()); + for (MetadataChangeProposal mcp : this.getMCPs(goldenFileStream)) { + emitter.emit(mcp); + } + emitter.close(); + goldenFileStream = ClassLoader.getSystemResourceAsStream("golden_files/mcps_golden.json"); + this.assertEqualJsonFile(goldenFileStream, outputFile); + + } + + private void assertEqualJsonFile(InputStream file1, String file2) throws StreamReadException, DatabindException, + IOException { + TypeReference>> typeRef = new TypeReference>>() { + }; + List> map1 = this.objectMapper.readValue(file1, typeRef); + File f2 = new File(file2); + List> map2 = this.objectMapper.readValue(f2, typeRef); + Assert.assertEquals(map1, map2); + } + + private List getMCPs(InputStream fileStream) throws StreamReadException, DatabindException, + IOException { + ArrayList mcps = new ArrayList(); + TypeReference[]> typeRef = new TypeReference[]>() { + }; + Map[] maps = this.objectMapper.readValue(fileStream, typeRef); + for (Map map : maps) { + String json = objectMapper.writeValueAsString(map); + DataMap data = dataTemplateCodec.stringToMap(json); + mcps.add(new MetadataChangeProposal(data)); + } + return mcps; + } + + @Test + public void testSuccessCallback() throws Exception { + + String tempRoot = tempFolder.getRoot().toString(); + String outputFile = tempRoot + "/testCallBack.json"; + FileEmitter emitter = new FileEmitter(FileEmitterConfig.builder().fileName(outputFile).build()); + MetadataChangeProposalWrapper mcpw = getMetadataChangeProposalWrapper("Test Dataset", "urn:li:dataset:foo"); + AtomicReference callbackResponse = new AtomicReference<>(); + Future future = emitter.emit(mcpw, new Callback() { + @Override + public void onCompletion(MetadataWriteResponse response) { + callbackResponse.set(response); + Assert.assertTrue(response.isSuccess()); + } + + @Override + public void onFailure(Throwable exception) { + Assert.fail("Should not be called"); + } + }); + + Assert.assertEquals(callbackResponse.get(), future.get()); + } + + @Test + public void testFailCallback() throws Exception { + + String tempRoot = tempFolder.getRoot().toString(); + String outputFile = tempRoot + "/testCallBack.json"; + FileEmitter emitter = new FileEmitter(FileEmitterConfig.builder().fileName(outputFile).build()); + emitter.close(); + MetadataChangeProposalWrapper mcpw = getMetadataChangeProposalWrapper("Test Dataset", "urn:li:dataset:foo"); + Future future = emitter.emit(mcpw, new Callback() { + @Override + public void onCompletion(MetadataWriteResponse response) { + + Assert.fail("Should not be called"); + } + + @Override + public void onFailure(Throwable exception) { + + } + }); + + Assert.assertFalse(future.get().isSuccess()); + + } + + private MetadataChangeProposalWrapper getMetadataChangeProposalWrapper(String description, String entityUrn) { + return MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn(entityUrn) + .upsert() + .aspect(new DatasetProperties().setDescription(description)) + .build(); + } + +} diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/AvroSerializerTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/AvroSerializerTest.java new file mode 100644 index 00000000000000..520594381426f5 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/AvroSerializerTest.java @@ -0,0 +1,58 @@ +package datahub.client.kafka; + +import com.linkedin.dataset.DatasetProperties; +import datahub.event.MetadataChangeProposalWrapper; +import java.io.File; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + + +public class AvroSerializerTest { + + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private MetadataChangeProposalWrapper getMetadataChangeProposalWrapper(String description, String entityUrn) { + return MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn(entityUrn) + .upsert() + .aspect(new DatasetProperties().setDescription(description)) + .build(); + } + + @Test + public void avroFileWrite() throws Exception { + + AvroSerializer avroSerializer = new AvroSerializer(); + File file = tempFolder.newFile("data.avro"); + DatumWriter writer = new GenericDatumWriter(avroSerializer.getRecordSchema()); + DataFileWriter dataFileWriter = new DataFileWriter(writer); + dataFileWriter.create(avroSerializer.getRecordSchema(), file); + String entityUrn = "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)"; + for (int i = 0; i < 10; ++i) { + MetadataChangeProposalWrapper metadataChangeProposalWrapper = getMetadataChangeProposalWrapper("Test description - " + i, entityUrn); + GenericRecord record = avroSerializer.serialize(metadataChangeProposalWrapper); + dataFileWriter.append(record); + } + dataFileWriter.close(); + + File readerFile = file; + DatumReader reader = new GenericDatumReader<>(avroSerializer.getRecordSchema()); + DataFileReader dataFileReader = new DataFileReader(readerFile, reader); + while (dataFileReader.hasNext()) { + GenericRecord record = dataFileReader.next(); + System.out.println(record.get("entityUrn")); + System.out.println(((GenericRecord) record.get("aspect")).get("value")); + } + } +} diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/KafkaEmitterTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/KafkaEmitterTest.java new file mode 100644 index 00000000000000..2645b114f97655 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/KafkaEmitterTest.java @@ -0,0 +1,105 @@ +package datahub.client.kafka; + +import static datahub.client.kafka.KafkaEmitter.DEFAULT_MCP_KAFKA_TOPIC; +import static java.util.Collections.singletonList; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Stream; + +import org.apache.avro.Schema; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.KafkaAdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.junit.Before; +import org.junit.Test; +import org.testcontainers.containers.Network; +import org.testcontainers.lifecycle.Startables; +import org.testng.Assert; + +import com.linkedin.dataset.DatasetProperties; + +import datahub.client.MetadataWriteResponse; +import datahub.client.kafka.containers.KafkaContainer; +import datahub.client.kafka.containers.SchemaRegistryContainer; +import datahub.client.kafka.containers.ZookeeperContainer; +import datahub.event.MetadataChangeProposalWrapper; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; + +public class KafkaEmitterTest { + + private static final String TOPIC = DEFAULT_MCP_KAFKA_TOPIC; + + private static Network network; + + private static ZookeeperContainer zookeeperContainer; + private static KafkaContainer kafkaContainer; + private static SchemaRegistryContainer schemaRegistryContainer; + private KafkaEmitterConfig config; + private KafkaEmitter emitter; + + @SuppressWarnings("resource") + @Before + public void confluentSetup() throws Exception { + network = Network.newNetwork(); + zookeeperContainer = new ZookeeperContainer().withNetwork(network); + kafkaContainer = new KafkaContainer(zookeeperContainer.getInternalUrl()).withNetwork(network); + schemaRegistryContainer = new SchemaRegistryContainer(zookeeperContainer.getInternalUrl()).withNetwork(network); + Startables.deepStart(Stream.of(zookeeperContainer, kafkaContainer, schemaRegistryContainer)).join(); + + createKafkaEmitter(); + createTopics(); + registerSchemaRegistryTypes(); + + } + + public void createKafkaEmitter() throws IOException { + KafkaEmitterConfig.KafkaEmitterConfigBuilder builder = KafkaEmitterConfig.builder(); + builder.bootstrap(kafkaContainer.getBootstrapServers()); + builder.schemaRegistryUrl(schemaRegistryContainer.getUrl()); + config = builder.build(); + emitter = new KafkaEmitter(config); + } + + @Test + public void testConnection() throws IOException, ExecutionException, InterruptedException { + Assert.assertTrue(emitter.testConnection()); + } + + @Test + public void testSend() throws IOException, InterruptedException, ExecutionException { + + @SuppressWarnings("rawtypes") + MetadataChangeProposalWrapper mcpw = getMetadataChangeProposalWrapper("Test Dataset", + "urn:li:dataset:(urn:li:dataPlatform:spark,foo.bar,PROD)"); + Future future = emitter.emit(mcpw); + MetadataWriteResponse response = future.get(); + System.out.println("Response: " + response); + Assert.assertTrue(response.isSuccess()); + } + + private AdminClient createAdminClient() { + return KafkaAdminClient.create(emitter.getKafkaConfgiProperties()); + } + + private static void registerSchemaRegistryTypes() throws IOException, RestClientException { + Schema mcpSchema = new AvroSerializer().getRecordSchema(); + CachedSchemaRegistryClient schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryContainer.getUrl(), 1000); + schemaRegistryClient.register(mcpSchema.getFullName(), mcpSchema); + } + + private void createTopics() throws InterruptedException, ExecutionException { + AdminClient adminClient = createAdminClient(); + short replicationFactor = 1; + int partitions = 1; + adminClient.createTopics(singletonList(new NewTopic(TOPIC, partitions, replicationFactor))).all().get(); + } + + @SuppressWarnings("rawtypes") + private MetadataChangeProposalWrapper getMetadataChangeProposalWrapper(String description, String entityUrn) { + return MetadataChangeProposalWrapper.builder().entityType("dataset").entityUrn(entityUrn).upsert() + .aspect(new DatasetProperties().setDescription(description)).build(); + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/KafkaContainer.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/KafkaContainer.java new file mode 100644 index 00000000000000..971782d2b7aa09 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/KafkaContainer.java @@ -0,0 +1,106 @@ +package datahub.client.kafka.containers; + +import com.github.dockerjava.api.command.InspectContainerResponse; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.TestcontainersConfiguration; + +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static datahub.client.kafka.containers.Utils.CONFLUENT_PLATFORM_VERSION; + +/** + * This container wraps Confluent Kafka. + * + */ +public class KafkaContainer extends GenericContainer { + + private static final String STARTER_SCRIPT = "/testcontainers_start.sh"; + + private static final int KAFKA_INTERNAL_PORT = 9092; + + public static final int KAFKA_INTERNAL_ADVERTISED_LISTENERS_PORT = 29092; + + private static final int PORT_NOT_ASSIGNED = -1; + + private String zookeeperConnect = null; + + private int port = PORT_NOT_ASSIGNED; + + private final String networkAlias = "kafka"; + + public KafkaContainer(String zookeeperConnect) { + this(CONFLUENT_PLATFORM_VERSION, zookeeperConnect); + } + + public KafkaContainer(String confluentPlatformVersion, String zookeeperConnect) { + super(getKafkaContainerImage(confluentPlatformVersion)); + + this.zookeeperConnect = zookeeperConnect; + withExposedPorts(KAFKA_INTERNAL_PORT); + + // Use two listeners with different names, it will force Kafka to communicate + // with itself via internal + // listener when KAFKA_INTER_BROKER_LISTENER_NAME is set, otherwise Kafka will + // try to use the advertised listener + withEnv("KAFKA_LISTENERS", + "PLAINTEXT://0.0.0.0:" + KAFKA_INTERNAL_ADVERTISED_LISTENERS_PORT + ",BROKER://0.0.0.0:" + KAFKA_INTERNAL_PORT); + withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT"); + withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER"); + + withEnv("KAFKA_BROKER_ID", "1"); + withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1"); + withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1"); + withEnv("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", Long.MAX_VALUE + ""); + withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0"); + + withNetworkAliases(networkAlias); + + } + + public String getBootstrapServers() { + if (port == PORT_NOT_ASSIGNED) { + throw new IllegalStateException("You should start Kafka container first"); + } + return String.format("PLAINTEXT://%s:%s", getHost(), port); + } + + @Override + protected void doStart() { + withCommand("sh", "-c", "while [ ! -f " + STARTER_SCRIPT + " ]; do sleep 0.1; done; " + STARTER_SCRIPT); + + super.doStart(); + } + + @Override + protected void containerIsStarting(InspectContainerResponse containerInfo, boolean reused) { + super.containerIsStarting(containerInfo, reused); + + port = getMappedPort(KAFKA_INTERNAL_PORT); + + if (reused) { + return; + } + + String command = "#!/bin/bash \n"; + command += "export KAFKA_ZOOKEEPER_CONNECT='" + zookeeperConnect + "'\n"; + command += "export KAFKA_ADVERTISED_LISTENERS='" + Stream + .concat(Stream.of("PLAINTEXT://" + networkAlias + ":" + KAFKA_INTERNAL_ADVERTISED_LISTENERS_PORT), + containerInfo.getNetworkSettings().getNetworks().values().stream() + .map(it -> "BROKER://" + it.getIpAddress() + ":" + KAFKA_INTERNAL_PORT)) + .collect(Collectors.joining(",")) + "'\n"; + + command += ". /etc/confluent/docker/bash-config \n"; + command += "/etc/confluent/docker/configure \n"; + command += "/etc/confluent/docker/launch \n"; + + copyFileToContainer(Transferable.of(command.getBytes(StandardCharsets.UTF_8), 700), STARTER_SCRIPT); + } + + private static String getKafkaContainerImage(String confluentPlatformVersion) { + return (String) TestcontainersConfiguration.getInstance().getProperties().getOrDefault("kafka.container.image", + "confluentinc/cp-kafka:" + confluentPlatformVersion); + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/SchemaRegistryContainer.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/SchemaRegistryContainer.java new file mode 100644 index 00000000000000..324e74d33e7045 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/SchemaRegistryContainer.java @@ -0,0 +1,47 @@ +package datahub.client.kafka.containers; + +import static datahub.client.kafka.containers.Utils.CONFLUENT_PLATFORM_VERSION; +import static java.lang.String.format; + +import java.io.IOException; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.TestcontainersConfiguration; + + +public class SchemaRegistryContainer extends GenericContainer { + private static final int SCHEMA_REGISTRY_INTERNAL_PORT = 8081; + + private final String networkAlias = "schema-registry"; + + public SchemaRegistryContainer(String zookeeperConnect) throws IOException { + this(CONFLUENT_PLATFORM_VERSION, zookeeperConnect); + } + + public SchemaRegistryContainer(String confluentPlatformVersion, String zookeeperConnect) throws IOException { + super(getSchemaRegistryContainerImage(confluentPlatformVersion)); + + addEnv("SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL", zookeeperConnect); + addEnv("SCHEMA_REGISTRY_HOST_NAME", "localhost"); + + withExposedPorts(SCHEMA_REGISTRY_INTERNAL_PORT); + withNetworkAliases(networkAlias); + + waitingFor(Wait.forHttp("/subjects")); + } + + public String getUrl() { + return format("http://%s:%d", this.getContainerIpAddress(), this.getMappedPort(SCHEMA_REGISTRY_INTERNAL_PORT)); + } + + + private static String getSchemaRegistryContainerImage(String confluentPlatformVersion) { + return (String) TestcontainersConfiguration + .getInstance().getProperties().getOrDefault( + "schemaregistry.container.image", + "confluentinc/cp-schema-registry:" + confluentPlatformVersion + ); + } +} + + diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java new file mode 100644 index 00000000000000..33f76e5bae90b0 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/Utils.java @@ -0,0 +1,23 @@ +package datahub.client.kafka.containers; + +import java.io.IOException; +import java.net.ServerSocket; + +final class Utils { + public static final String CONFLUENT_PLATFORM_VERSION = "5.3.1"; + + private Utils() { + } + + /** + * Retrieves a random port that is currently not in use on this machine. + * + * @return a free port + * @throws IOException wraps the exceptions which may occur during this method call. + */ + static int getRandomFreePort() throws IOException { + @SuppressWarnings("resource") + ServerSocket serverSocket = new ServerSocket(0); + return serverSocket.getLocalPort(); + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/ZookeeperContainer.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/ZookeeperContainer.java new file mode 100644 index 00000000000000..62ee3765a39b06 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/kafka/containers/ZookeeperContainer.java @@ -0,0 +1,46 @@ +package datahub.client.kafka.containers; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.TestcontainersConfiguration; + +import java.io.IOException; +import java.util.HashMap; + +import static datahub.client.kafka.containers.Utils.CONFLUENT_PLATFORM_VERSION; +import static java.lang.String.format; + +public class ZookeeperContainer extends GenericContainer { + + private static final int ZOOKEEPER_INTERNAL_PORT = 2181; + private static final int ZOOKEEPER_TICK_TIME = 2000; + + private final String networkAlias = "zookeeper"; + + public ZookeeperContainer() throws IOException { + this(CONFLUENT_PLATFORM_VERSION); + } + + public ZookeeperContainer(String confluentPlatformVersion) throws IOException { + super(getZookeeperContainerImage(confluentPlatformVersion)); + + HashMap env = new HashMap(); + env.put("ZOOKEEPER_CLIENT_PORT", Integer.toString(ZOOKEEPER_INTERNAL_PORT)); + env.put("ZOOKEEPER_TICK_TIME", Integer.toString(ZOOKEEPER_TICK_TIME)); + withEnv(env); + + addExposedPort(ZOOKEEPER_INTERNAL_PORT); + withNetworkAliases(networkAlias); + } + + public String getInternalUrl() { + return format("%s:%d", networkAlias, ZOOKEEPER_INTERNAL_PORT); + } + + private static String getZookeeperContainerImage(String confluentPlatformVersion) { + return (String) TestcontainersConfiguration + .getInstance().getProperties().getOrDefault( + "zookeeper.container.image", + "confluentinc/cp-zookeeper:" + confluentPlatformVersion + ); + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java index c2a2b852219360..7a8c95da2ffedb 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/rest/RestEmitterTest.java @@ -27,7 +27,11 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; + +import javax.net.ssl.SSLHandshakeException; + import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.concurrent.FutureCallback; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; @@ -380,4 +384,30 @@ public void testUserAgentHeader() throws IOException, ExecutionException, Interr request("/config") .withHeader("User-Agent", "DataHub-RestClient/" + version)); } + + @Test + public void testDisableSslVerification() throws IOException, InterruptedException, ExecutionException { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().disableSslVerification(true).build()); + final String hostWithSsl = "https://self-signed.badssl.com"; + final HttpGet request = new HttpGet(hostWithSsl); + + final HttpResponse response = restEmitter.getHttpClient().execute(request, null).get(); + restEmitter.close(); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + } + + @Test + public void testSslVerificationException() throws IOException, InterruptedException, ExecutionException { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().disableSslVerification(false).build()); + final String hostWithSsl = "https://self-signed.badssl.com"; + final HttpGet request = new HttpGet(hostWithSsl); + try { + HttpResponse response = restEmitter.getHttpClient().execute(request, null).get(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ExecutionException); + Assert.assertTrue(((ExecutionException) e).getCause() instanceof SSLHandshakeException); + } + restEmitter.close(); + } } \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/src/test/resources/golden_files/mcps_golden.json b/metadata-integration/java/datahub-client/src/test/resources/golden_files/mcps_golden.json new file mode 100644 index 00000000000000..679e6b87327c4f --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/resources/golden_files/mcps_golden.json @@ -0,0 +1,2037 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:sales\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"datacharmer\"}, \"name\": \"datacharmer\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "employees", + "qualifiedName": null, + "description": "这是一个很好的描述", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "datacharmer.employees", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "emp_no", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "birth_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=14)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=16)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "gender", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.EnumType": {} + } + }, + "nativeDataType": "ENUM('M', 'F')", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "hire_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "salaries", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "datacharmer.salaries", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "emp_no", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "salary", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "from_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "to_date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "metadata_aspect", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "metagalaxy.metadata_aspect", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "urn", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=500)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "aspect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=200)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "version", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "BIGINT()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "metadata", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "LONGTEXT()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "createdon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME(fsp=6)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "createdby", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=255)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "createdfor", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=255)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:sales\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "metadata_index", + "qualifiedName": null, + "description": "This is a table comment", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "metagalaxy.metadata_index", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "BIGINT()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "urn", + "jsonPath": null, + "nullable": false, + "description": "This is a column comment about URNs", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=200)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "aspect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=150)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "path", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=150)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "longVal", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "BIGINT()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "stringVal", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=200)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "doubleVal", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "DOUBLE(asdecimal=True)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:sales\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `metadata_index_view` AS select `metadata_index`.`id` AS `id`,`metadata_index`.`urn` AS `urn`,`metadata_index`.`path` AS `path`,`metadata_index`.`doubleVal` AS `doubleVal` from `metadata_index`", + "is_view": "True" + }, + "externalUrl": null, + "name": "metadata_index_view", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "metagalaxy.metadata_index_view", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "BIGINT()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "urn", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=200)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "path", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=150)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "doubleVal", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "DOUBLE(asdecimal=True)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `metadata_index_view` AS select `metadata_index`.`id` AS `id`,`metadata_index`.`urn` AS `urn`,`metadata_index`.`path` AS `path`,`metadata_index`.`doubleVal` AS `doubleVal` from `metadata_index`\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "value": "{\"domains\": [\"urn:li:domain:sales\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"northwind\"}, \"name\": \"northwind\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "northwind.customers", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "company", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "email_address", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "priority", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "northwind.orders", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "description", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "fk_order_customer", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)" + } + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"test_cases\"}, \"name\": \"test_cases\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:989c003cbe689094c2b5c340a67f62be\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "test_empty", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test_cases.test_empty", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "dummy", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 10, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"emp_no\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"10001\", \"10002\", \"10003\", \"10004\", \"10005\", \"10006\", \"10007\", \"10008\", \"10009\", \"10010\"]}, {\"fieldPath\": \"birth_date\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1952-04-19\", \"max\": \"1964-06-02\", \"sampleValues\": [\"1953-09-02\", \"1964-06-02\", \"1959-12-03\", \"1954-05-01\", \"1955-01-21\", \"1953-04-20\", \"1957-05-23\", \"1958-02-19\", \"1952-04-19\", \"1963-06-01\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Georgi\", \"Bezalel\", \"Parto\", \"Chirstian\", \"Kyoichi\", \"Anneke\", \"Tzvetan\", \"Saniya\", \"Sumant\", \"Duangkaew\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Facello\", \"Simmel\", \"Bamford\", \"Koblick\", \"Maliniak\", \"Preusig\", \"Zielinski\", \"Kalloufi\", \"Peac\", \"Piveteau\"]}, {\"fieldPath\": \"gender\", \"uniqueCount\": 2, \"uniqueProportion\": 0.2, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"M\", \"frequency\": 5}, {\"value\": \"F\", \"frequency\": 5}], \"sampleValues\": [\"M\", \"F\", \"M\", \"M\", \"M\", \"F\", \"F\", \"M\", \"F\", \"F\"]}, {\"fieldPath\": \"hire_date\", \"uniqueCount\": 10, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1985-02-18\", \"max\": \"1994-09-15\", \"sampleValues\": [\"1986-06-26\", \"1985-11-21\", \"1986-08-28\", \"1986-12-01\", \"1989-09-12\", \"1989-06-02\", \"1989-02-10\", \"1994-09-15\", \"1985-02-18\", \"1989-08-24\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 112, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"emp_no\", \"uniqueCount\": 10, \"uniqueProportion\": 0.08928571428571429, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"10001\", \"max\": \"10010\", \"mean\": \"10005.3125\", \"median\": \"10005.0\", \"stdev\": \"2.834889609688869\", \"distinctValueFrequencies\": [{\"value\": \"10001\", \"frequency\": 17}, {\"value\": \"10002\", \"frequency\": 6}, {\"value\": \"10003\", \"frequency\": 7}, {\"value\": \"10004\", \"frequency\": 16}, {\"value\": \"10005\", \"frequency\": 13}, {\"value\": \"10006\", \"frequency\": 12}, {\"value\": \"10007\", \"frequency\": 14}, {\"value\": \"10008\", \"frequency\": 3}, {\"value\": \"10009\", \"frequency\": 18}, {\"value\": \"10010\", \"frequency\": 6}], \"sampleValues\": [\"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10001\", \"10002\", \"10002\", \"10002\"]}, {\"fieldPath\": \"salary\", \"uniqueCount\": 111, \"uniqueProportion\": 0.9910714285714286, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"40000\", \"max\": \"94692\", \"mean\": \"68303.11607142857\", \"median\": \"69544.0\", \"stdev\": \"15505.291475014095\", \"sampleValues\": [\"60117\", \"62102\", \"66074\", \"66596\", \"66961\", \"71046\", \"74333\", \"75286\", \"75994\", \"76884\", \"80013\", \"81025\", \"81097\", \"84917\", \"85112\", \"85097\", \"88958\", \"65909\", \"65909\", \"67534\"]}, {\"fieldPath\": \"from_date\", \"uniqueCount\": 106, \"uniqueProportion\": 0.9464285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1985-02-18\", \"max\": \"2002-06-22\", \"sampleValues\": [\"1986-06-26\", \"1987-06-26\", \"1988-06-25\", \"1989-06-25\", \"1990-06-25\", \"1991-06-25\", \"1992-06-24\", \"1993-06-24\", \"1994-06-24\", \"1995-06-24\", \"1996-06-23\", \"1997-06-23\", \"1998-06-23\", \"1999-06-23\", \"2000-06-22\", \"2001-06-22\", \"2002-06-22\", \"1996-08-03\", \"1997-08-03\", \"1998-08-03\"]}, {\"fieldPath\": \"to_date\", \"uniqueCount\": 99, \"uniqueProportion\": 0.8839285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1986-02-18\", \"max\": \"9999-01-01\", \"sampleValues\": [\"1987-06-26\", \"1988-06-25\", \"1989-06-25\", \"1990-06-25\", \"1991-06-25\", \"1992-06-24\", \"1993-06-24\", \"1994-06-24\", \"1995-06-24\", \"1996-06-23\", \"1997-06-23\", \"1998-06-23\", \"1999-06-23\", \"2000-06-22\", \"2001-06-22\", \"2002-06-22\", \"9999-01-01\", \"1997-08-03\", \"1998-08-03\", \"1999-08-03\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 5, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"1\", \"2\", \"3\", \"4\", \"5\"]}, {\"fieldPath\": \"company\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Company A\", \"Company B\", \"Company C\", \"Company D\", \"Company E\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Axen\", \"Bedecs\", \"Donnell\", \"Gratacos Solsona\", \"Lee\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 5, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Anna\", \"Antonio\", \"Christina\", \"Martin\", \"Thomas\"]}, {\"fieldPath\": \"email_address\", \"uniqueCount\": 0, \"nullCount\": 5, \"nullProportion\": 1.0, \"sampleValues\": []}, {\"fieldPath\": \"priority\", \"uniqueCount\": 3, \"uniqueProportion\": 0.75, \"nullCount\": 1, \"nullProportion\": 0.2, \"min\": \"3.8\", \"max\": \"4.9\", \"mean\": \"4.175000011920929\", \"median\": \"4.0\", \"distinctValueFrequencies\": [{\"value\": \"3.8\", \"frequency\": 1}, {\"value\": \"4.0\", \"frequency\": 2}, {\"value\": \"4.9\", \"frequency\": 1}], \"sampleValues\": [\"4.0\", \"4.9\", \"4.0\", \"3.8\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 3, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"description\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"customer_id\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,test_cases.test_empty,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 1, \"fieldProfiles\": [{\"fieldPath\": \"dummy\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-integration/java/datahub-protobuf/README.md b/metadata-integration/java/datahub-protobuf/README.md index e6b9465abd5ee8..e7e4bd23a00853 100644 --- a/metadata-integration/java/datahub-protobuf/README.md +++ b/metadata-integration/java/datahub-protobuf/README.md @@ -444,7 +444,7 @@ The following is a consolidated example for the possible field level term option ##### OWNER -One or more owners can be specified and can be any combination of `corpUser` and `corpGroup` entities. The default entity type is `corpGroup`. By default, the ownership type is set to `producer`, see the second example for setting the ownership type. +One or more owners can be specified and can be any combination of `corpUser` and `corpGroup` entities. The default entity type is `corpGroup`. By default, the ownership type is set to `technical_owner`, see the second example for setting the ownership type. The following example assigns the ownership to a group of `myGroup` and a user called `myName`. diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/OwnershipVisitor.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/OwnershipVisitor.java index 0f0ad2ea69b3bc..7bb4d9860f72cc 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/OwnershipVisitor.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/dataset/OwnershipVisitor.java @@ -37,14 +37,14 @@ public Stream visitGraph(VisitContext context) { try { ownershipType = OwnershipType.valueOf(entry.getKey().getName().toUpperCase()); } catch (IllegalArgumentException e) { - ownershipType = OwnershipType.PRODUCER; + ownershipType = OwnershipType.TECHNICAL_OWNER; } String[] id = entry.getValue().toLowerCase().split(":", 2); return new Owner() .setType(ownershipType) .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL)) - .setOwner(new Urn(id.length > 1 ? id[0] : "corpgroup", id[id.length - 1])); + .setOwner(new Urn(id.length > 1 ? id[0].replaceFirst("corpgroup", "corpGroup") : "corpGroup", id[id.length - 1])); } catch (URISyntaxException e) { System.err.println(e.getMessage()); return null; diff --git a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java index baefd365c8d4bd..cf2649e86dc43a 100644 --- a/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java +++ b/metadata-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java @@ -27,17 +27,17 @@ public void visitorTest() throws IOException { OwnershipVisitor test = new OwnershipVisitor(); assertEquals(Set.of(new Owner() - .setType(OwnershipType.PRODUCER) + .setType(OwnershipType.TECHNICAL_OWNER) .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL)) - .setOwner(Urn.createFromTuple("corpgroup", "teamb")), + .setOwner(Urn.createFromTuple("corpGroup", "teamb")), new Owner() - .setType(OwnershipType.PRODUCER) + .setType(OwnershipType.TECHNICAL_OWNER) .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL)) .setOwner(Urn.createFromTuple("corpuser", "datahub")), new Owner() .setType(OwnershipType.TECHNICAL_OWNER) .setSource(new OwnershipSource().setType(OwnershipSourceType.MANUAL)) - .setOwner(Urn.createFromTuple("corpgroup", "technicalowner")) + .setOwner(Urn.createFromTuple("corpGroup", "technicalowner")) ), graph.accept(getVisitContextBuilder("extended_protobuf.MessageA"), List.of(test)).collect(Collectors.toSet())); } diff --git a/metadata-integration/java/spark-lineage/README.md b/metadata-integration/java/spark-lineage/README.md index 27cac573d6a21a..05bd2f973e810e 100644 --- a/metadata-integration/java/spark-lineage/README.md +++ b/metadata-integration/java/spark-lineage/README.md @@ -1,26 +1,41 @@ # Spark + To integrate Spark with DataHub, we provide a lightweight Java agent that listens for Spark application and job events and pushes metadata out to DataHub in real-time. The agent listens to events such application start/end, and SQLExecution start/end to create pipelines (i.e. DataJob) and tasks (i.e. DataFlow) in Datahub along with lineage to datasets that are being read from and written to. Read on to learn how to configure this for different Spark scenarios. ## Configuring Spark agent + The Spark agent can be configured using a config file or while creating a spark Session. ## Before you begin: Versions and Release Notes + Versioning of the jar artifact will follow the semantic versioning of the main [DataHub repo](https://github.com/datahub-project/datahub) and release notes will be available [here](https://github.com/datahub-project/datahub/releases). Always check [the Maven central repository](https://search.maven.org/search?q=a:datahub-spark-lineage) for the latest released version. ### Configuration Instructions: spark-submit -When running jobs using spark-submit, the agent needs to be configured in the config file. -``` -spark.master spark://spark-master:7077 +When running jobs using spark-submit, the agent needs to be configured in the config file. +```text #Configuring datahub spark agent jar -spark.jars.packages io.acryl:datahub-spark-lineage:0.8.23 +spark.jars.packages io.acryl:datahub-spark-lineage:0.8.23 spark.extraListeners datahub.spark.DatahubSparkListener spark.datahub.rest.server http://localhost:8080 ``` +#### Configuration for Amazon EMR + +Set the following spark-defaults configuration properties as it stated [here](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-configure.html) + +``` +spark.jars.packages io.acryl:datahub-spark-lineage:0.8.23 +spark.extraListeners datahub.spark.DatahubSparkListener +spark.datahub.rest.server https://your_datahub_host/gms +#If you have authentication set up then you also need to specify the Datahub access token +spark.datahub.rest.token yourtoken +``` + ### Configuration Instructions: Notebooks + When running interactive jobs from a notebook, the listener can be configured while building the Spark Session. ```python @@ -35,7 +50,8 @@ spark = SparkSession.builder \ ``` ### Configuration Instructions: Standalone Java Applications -The configuration for standalone Java apps is very similar. + +The configuration for standalone Java apps is very similar. ```java spark = SparkSession.builder() @@ -54,15 +70,15 @@ spark = SparkSession.builder() |-------------------------------------------------|----------|---------|-------------------------------------------------------------------------| | spark.jars.packages | ✅ | | Set with latest/required version io.acryl:datahub-spark-lineage:0.8.23 | | spark.extraListeners | ✅ | | datahub.spark.DatahubSparkListener | -| spark.datahub.rest.server | ✅ | | Datahub server url eg:http://localhost:8080 | +| spark.datahub.rest.server | ✅ | | Datahub server url eg: | | spark.datahub.rest.token | | | Authentication token. | +| spark.datahub.rest.disable_ssl_verification | | false | Disable SSL certificate validation. Caution: Only use this if you know what you are doing! | | spark.datahub.metadata.pipeline.platformInstance| | | Pipeline level platform instance | | spark.datahub.metadata.dataset.platformInstance| | | dataset level platform instance | | spark.datahub.metadata.dataset.env | | PROD | [Supported values](https://datahubproject.io/docs/graphql/enums#fabrictype). In all other cases, will fallback to PROD | | spark.datahub.coalesce_jobs | | false | Only one datajob(taask) will be emitted containing all input and output datasets for the spark application | | spark.datahub.parent.datajob_urn | | | Specified dataset will be set as upstream dataset for datajob created. Effective only when spark.datahub.coalesce_jobs is set to true | - ## What to Expect: The Metadata Model As of current writing, the Spark agent produces metadata related to the Spark job, tasks and lineage edges to datasets. @@ -71,23 +87,26 @@ As of current writing, the Spark agent produces metadata related to the Spark jo - A task is created per unique Spark query execution within an app. ### Custom properties & relating to Spark UI + The following custom properties in pipelines and tasks relate to the Spark UI: + - appName and appId in a pipeline can be used to determine the Spark application - description and SQLQueryId in a task can be used to determine the Query Execution within the application on the SQL tab of Spark UI -Other custom properties of pipelines and tasks capture the start and end times of execution etc. +Other custom properties of pipelines and tasks capture the start and end times of execution etc. The query plan is captured in the *queryPlan* property of a task. - - ### Spark versions supported + The primary version tested is Spark/Scala version 2.4.8/2_11. This library has also been tested to work with Spark versions(2.2.0 - 2.4.8) and Scala versions(2.10 - 2.12). For the Spark 3.x series, this has been tested to work with Spark 3.1.2 and 3.2.0 with Scala 2.12. Other combinations are not guaranteed to work currently. Support for other Spark versions is planned in the very near future. ### Environments tested with + This initial release has been tested with the following environments: + - spark-submit of Python/Java applications to local and remote servers - Jupyter notebooks with pyspark code - Standalone Java applications @@ -97,6 +116,7 @@ Note that testing for other environments such as Databricks is planned in near f ### Spark commands supported Below is a list of Spark commands that are parsed currently: + - InsertIntoHadoopFsRelationCommand - SaveIntoDataSourceCommand (jdbc) - CreateHiveTableAsSelectCommand @@ -105,12 +125,14 @@ Below is a list of Spark commands that are parsed currently: Effectively, these support data sources/sinks corresponding to Hive, HDFS and JDBC. DataFrame.persist command is supported for below LeafExecNodes: + - FileSourceScanExec - HiveTableScanExec - RowDataSourceScanExec - InMemoryTableScanExec ### Spark commands not yet supported + - View related commands - Cache commands and implications on lineage - RDD jobs @@ -127,21 +149,28 @@ DataFrame.persist command is supported for below LeafExecNodes: - Following info logs are generated On Spark context startup -``` + +```text YY/MM/DD HH:mm:ss INFO DatahubSparkListener: DatahubSparkListener initialised. YY/MM/DD HH:mm:ss INFO SparkContext: Registered listener datahub.spark.DatahubSparkListener ``` + On application start + ``` YY/MM/DD HH:mm:ss INFO DatahubSparkListener: Application started: SparkListenerApplicationStart(AppName,Some(local-1644489736794),1644489735772,user,None,None) YY/MM/DD HH:mm:ss INFO McpEmitter: REST Emitter Configuration: GMS url YY/MM/DD HH:mm:ss INFO McpEmitter: REST Emitter Configuration: Token XXXXX ``` + On pushing data to server + ``` YY/MM/DD HH:mm:ss INFO McpEmitter: MetadataWriteResponse(success=true, responseContent={"value":""}, underlyingResponse=HTTP/1.1 200 OK [Date: day, DD month year HH:mm:ss GMT, Content-Type: application/json, X-RestLi-Protocol-Version: 2.0.0, Content-Length: 97, Server: Jetty(9.4.46.v20220331)] [Content-Length: 97,Chunked: false]) ``` + On application end + ``` YY/MM/DD HH:mm:ss INFO DatahubSparkListener: Application ended : AppName AppID ``` @@ -154,6 +183,7 @@ log4j.logger.datahub.client.rest=DEBUG ``` ## Known limitations + - Only postgres supported for JDBC sources in this initial release. Support for other driver URL formats will be added in future. - Behavior with cached datasets is not fully specified/defined in context of lineage. - There is a possibility that very short-lived jobs that run within a few milliseconds may not be captured by the listener. This should not cause an issue for realistic Spark applications. diff --git a/metadata-integration/java/spark-lineage/build.gradle b/metadata-integration/java/spark-lineage/build.gradle index 13765a8aa78e59..f04878d5bf5c28 100644 --- a/metadata-integration/java/spark-lineage/build.gradle +++ b/metadata-integration/java/spark-lineage/build.gradle @@ -30,7 +30,6 @@ if (project.hasProperty("releaseVersion")) { try { // apply this plugin in a try-catch block so that we can handle cases without .git directory apply plugin: "com.palantir.git-version" - println("In else section") def details = versionDetails() detailedVersionString = gitVersion() version = details.lastTag diff --git a/metadata-integration/java/spark-lineage/scripts/check_jar.sh b/metadata-integration/java/spark-lineage/scripts/check_jar.sh index 59d104b78c93b1..430d89109d1e2c 100755 --- a/metadata-integration/java/spark-lineage/scripts/check_jar.sh +++ b/metadata-integration/java/spark-lineage/scripts/check_jar.sh @@ -24,7 +24,16 @@ jar -tvf $jarFile |\ grep -v "xml-header-style.xml" |\ grep -v "license.header" |\ grep -v "module-info.class" |\ - grep -v "client.properties" + grep -v "com/google/" |\ + grep -v "org/codehaus/" |\ + grep -v "client.properties" |\ + grep -v "kafka" |\ + grep -v "win/" |\ + grep -v "include/" |\ + grep -v "linux/" |\ + grep -v "darwin" |\ + grep -v "MetadataChangeProposal.avsc" |\ + grep -v "aix" if [ $? -ne 0 ]; then echo "✅ No unexpected class paths found in ${jarFile}" diff --git a/metadata-integration/java/spark-lineage/src/main/java/datahub/spark/consumer/impl/McpEmitter.java b/metadata-integration/java/spark-lineage/src/main/java/datahub/spark/consumer/impl/McpEmitter.java index f8f1bafc230c88..336246fa9d3e89 100644 --- a/metadata-integration/java/spark-lineage/src/main/java/datahub/spark/consumer/impl/McpEmitter.java +++ b/metadata-integration/java/spark-lineage/src/main/java/datahub/spark/consumer/impl/McpEmitter.java @@ -26,7 +26,7 @@ public class McpEmitter implements LineageConsumer { private static final String TRANSPORT_KEY = "transport"; private static final String GMS_URL_KEY = "rest.server"; private static final String GMS_AUTH_TOKEN = "rest.token"; - + private static final String DISABLE_SSL_VERIFICATION_KEY = "rest.disable_ssl_verification"; private Optional getEmitter() { Optional emitter = Optional.empty(); switch (emitterType) { @@ -78,12 +78,19 @@ public McpEmitter(Config datahubConf) { String gmsUrl = datahubConf.hasPath(GMS_URL_KEY) ? datahubConf.getString(GMS_URL_KEY) : "http://localhost:8080"; String token = datahubConf.hasPath(GMS_AUTH_TOKEN) ? datahubConf.getString(GMS_AUTH_TOKEN) : null; + boolean disableSslVerification = datahubConf.hasPath(DISABLE_SSL_VERIFICATION_KEY) ? datahubConf.getBoolean( + DISABLE_SSL_VERIFICATION_KEY) : false; log.info("REST Emitter Configuration: GMS url {}{}", gmsUrl, (datahubConf.hasPath(GMS_URL_KEY) ? "" : "(default)")); if (token != null) { log.info("REST Emitter Configuration: Token {}", (token != null) ? "XXXXX" : "(empty)"); } - restEmitterConfig = Optional.of(RestEmitterConfig.builder().server(gmsUrl).token(token).build()); + if (disableSslVerification) { + log.warn("REST Emitter Configuration: ssl verification will be disabled."); + } + restEmitterConfig = Optional.of(RestEmitterConfig.builder() + .server(gmsUrl).token(token) + .disableSslVerification(disableSslVerification).build()); break; default: diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java index ba8898bb65a01b..ceced5dd83ca94 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityService.java @@ -220,29 +220,73 @@ private void deleteReference(final Urn urn, final RelatedEntity relatedEntity) { updatedAspect.get(), aspectSpec.getPegasusSchema(), path)); }); - // If there has been an update - if (!updatedAspect.get().equals(aspect)) { - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - proposal.setEntityUrn(relatedUrn); - proposal.setChangeType(ChangeType.UPSERT); - proposal.setEntityType(relatedUrn.getEntityType()); - proposal.setAspectName(aspectName); - proposal.setAspect(GenericRecordUtils.serializeAspect(updatedAspect.get())); - - final AuditStamp auditStamp = new AuditStamp().setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); - final EntityService.IngestProposalResult ingestProposalResult = _entityService.ingestProposal(proposal, auditStamp); - - if (!ingestProposalResult.isDidUpdate()) { - log.error("Failed to ingest aspect with references removed. Before {}, after: {}, please check MCP processor" - + " logs for more information", aspect, updatedAspect); - handleError(new DeleteEntityServiceError("Failed to ingest new aspect", - DeleteEntityServiceErrorReason.MCP_PROCESSOR_FAILED, - ImmutableMap.of("proposal", proposal))); + // If there has been an update, then we produce an MCE. + if (!aspect.equals(updatedAspect.get())) { + if (updatedAspect.get() == null) { + // Then we should remove the aspect. + deleteAspect(relatedUrn, aspectName, aspect); + } else { + // Then we should update the aspect. + updateAspect(relatedUrn, aspectName, aspect, updatedAspect.get()); } } }); } + /** + * Delete an existing aspect for an urn. + * + * @param urn the urn of the entity to remove the aspect for + * @param aspectName the aspect to remove + * @param prevAspect the old value for the aspect + */ + private void deleteAspect(Urn urn, String aspectName, RecordTemplate prevAspect) { + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(urn); + proposal.setChangeType(ChangeType.DELETE); + proposal.setEntityType(urn.getEntityType()); + proposal.setAspectName(aspectName); + + final AuditStamp auditStamp = new AuditStamp().setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + final EntityService.IngestProposalResult ingestProposalResult = _entityService.ingestProposal(proposal, auditStamp); + + if (!ingestProposalResult.isDidUpdate()) { + log.error("Failed to ingest aspect with references removed. Before {}, after: null, please check MCP processor" + + " logs for more information", prevAspect); + handleError(new DeleteEntityServiceError("Failed to ingest new aspect", + DeleteEntityServiceErrorReason.MCP_PROCESSOR_FAILED, + ImmutableMap.of("proposal", proposal))); + } + } + + /** + * Update an aspect for an urn. + * + * @param urn the urn of the entity to remove the aspect for + * @param aspectName the aspect to remove + * @param prevAspect the old value for the aspect + * @param newAspect the new value for the aspect + */ + private void updateAspect(Urn urn, String aspectName, RecordTemplate prevAspect, RecordTemplate newAspect) { + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(urn); + proposal.setChangeType(ChangeType.UPSERT); + proposal.setEntityType(urn.getEntityType()); + proposal.setAspectName(aspectName); + proposal.setAspect(GenericRecordUtils.serializeAspect(newAspect)); + + final AuditStamp auditStamp = new AuditStamp().setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + final EntityService.IngestProposalResult ingestProposalResult = _entityService.ingestProposal(proposal, auditStamp); + + if (!ingestProposalResult.isDidUpdate()) { + log.error("Failed to ingest aspect with references removed. Before {}, after: {}, please check MCP processor" + + " logs for more information", prevAspect, newAspect); + handleError(new DeleteEntityServiceError("Failed to ingest new aspect", + DeleteEntityServiceErrorReason.MCP_PROCESSOR_FAILED, + ImmutableMap.of("proposal", proposal))); + } + } + /** * Utility method that attempts to find Aspect information as well as the associated path spec for a given urn that diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java index 1036ddda5e9c86..58b5341c4ae0cd 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/DeleteEntityUtils.java @@ -50,6 +50,10 @@ public static Aspect getAspectWithReferenceRemoved(String value, RecordTemplate try { final DataMap copy = aspect.copy().data(); final DataComplex newValue = removeValueBasedOnPath(value, schema, copy, aspectPath.getPathComponents(), 0); + if (newValue == null) { + // If the new value is null, we should remove the aspect. + return null; + } return new Aspect((DataMap) newValue); } catch (CloneNotSupportedException e) { return new Aspect(); @@ -105,7 +109,8 @@ private static DataComplex removeValueFromMap(String value, RecordDataSchema spe if (valueExistsInRecord) { if (canDelete) { record.remove(pathComponents.get(index)); - } else if (record.size() == 1) { + } else { + // If the field is required, then we need to remove the entire record (if possible) return null; } } else { @@ -126,6 +131,10 @@ private static DataComplex removeValueFromMap(String value, RecordDataSchema spe record.remove(key); } else if (record.size() == 1) { return null; + } else { + // Not optional and not the only field, then this is a bad delete. Need to throw. + throw new UnsupportedOperationException( + String.format("Delete failed! Failed to field with name %s from DataMap. The field is required!", key)); } } else { record.put(key, result); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspectIdentifier.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspectIdentifier.java index 00919335516711..cb360192c01201 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspectIdentifier.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityAspectIdentifier.java @@ -1,16 +1,28 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.entity.cassandra.CassandraAspect; +import com.linkedin.metadata.entity.ebean.EbeanAspectV2; +import javax.annotation.Nonnull; import lombok.Value; +import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nonnull; /** * This class holds values required to construct a unique key to identify an entity aspect record in a database. * Its existence started mainly for compatibility with {@link com.linkedin.metadata.entity.ebean.EbeanAspectV2.PrimaryKey} */ @Value +@Slf4j public class EntityAspectIdentifier { @Nonnull String urn; @Nonnull String aspect; long version; + + public static EntityAspectIdentifier fromEbean(EbeanAspectV2 ebeanAspectV2) { + return new EntityAspectIdentifier(ebeanAspectV2.getUrn(), ebeanAspectV2.getAspect(), ebeanAspectV2.getVersion()); + } + + public static EntityAspectIdentifier fromCassandra(CassandraAspect cassandraAspect) { + return new EntityAspectIdentifier(cassandraAspect.getUrn(), cassandraAspect.getAspect(), cassandraAspect.getVersion()); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index a4b61d036c6ef1..0c20e3fea51949 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -3,7 +3,6 @@ import com.codahale.metrics.Timer; import com.datahub.util.RecordUtils; import com.datahub.util.exception.ModelConversionException; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterators; @@ -44,15 +43,10 @@ import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.MetadataAuditOperation; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; -import lombok.Value; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.net.URISyntaxException; import java.sql.Timestamp; import java.util.ArrayList; @@ -67,11 +61,13 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.Constants.ASPECT_LATEST_VERSION; -import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; -import static com.linkedin.metadata.utils.PegasusUtils.getDataTemplateClassFromSchema; -import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.utils.PegasusUtils.*; /** @@ -726,6 +722,17 @@ protected RecordTemplate sendEventForUpdateAspectResult(@Nonnull final Urn urn, return updatedValue; } + /** + * Ingest a new {@link MetadataChangeProposal}. Note that this method does NOT include any additional aspects or do any + * enrichment, instead it changes only those which are provided inside the metadata change proposal. + * + * Do not use this method directly for creating new entities, as it DOES NOT create an Entity Key aspect in the DB. Instead, + * use an Entity Client. + * + * @param metadataChangeProposal the proposal to ingest + * @param auditStamp an audit stamp representing the time and actor proposing the change + * @return an {@link IngestProposalResult} containing the results + */ public IngestProposalResult ingestProposal(@Nonnull MetadataChangeProposal metadataChangeProposal, AuditStamp auditStamp) { @@ -1039,10 +1046,10 @@ protected Map> getLatestAspectUnions( } /** - Returns true if entityType should have some aspect as per its definition + Returns true if entityType should have some aspect as per its definition but aspects given does not have that aspect */ - private boolean isAspectProvided(String entityType, String aspectName, Set aspects) { + private boolean isAspectMissing(String entityType, String aspectName, Set aspects) { return _entityRegistry.getEntitySpec(entityType).getAspectSpecMap().containsKey(aspectName) && !aspects.contains(aspectName); } @@ -1053,21 +1060,16 @@ public List> generateDefaultAspectsIfMissing(@Nonnu Set aspectsToGet = new HashSet<>(); String entityType = urnToEntityName(urn); - boolean shouldCheckBrowsePath = isAspectProvided(entityType, BROWSE_PATHS, includedAspects); + boolean shouldCheckBrowsePath = isAspectMissing(entityType, BROWSE_PATHS, includedAspects); if (shouldCheckBrowsePath) { aspectsToGet.add(BROWSE_PATHS); } - boolean shouldCheckDataPlatform = isAspectProvided(entityType, DATA_PLATFORM_INSTANCE, includedAspects); + boolean shouldCheckDataPlatform = isAspectMissing(entityType, DATA_PLATFORM_INSTANCE, includedAspects); if (shouldCheckDataPlatform) { aspectsToGet.add(DATA_PLATFORM_INSTANCE); } - boolean shouldHaveStatusSet = isAspectProvided(entityType, STATUS, includedAspects); - if (shouldHaveStatusSet) { - aspectsToGet.add(STATUS); - } - List> aspects = new ArrayList<>(); final String keyAspectName = getKeyAspectName(urn); aspectsToGet.add(keyAspectName); @@ -1096,12 +1098,6 @@ public List> generateDefaultAspectsIfMissing(@Nonnu .ifPresent(aspect -> aspects.add(Pair.of(DATA_PLATFORM_INSTANCE, aspect))); } - if (shouldHaveStatusSet && latestAspects.get(STATUS) != null) { - Status status = new Status(); - status.setRemoved(false); - aspects.add(Pair.of(STATUS, status)); - } - return aspects; } @@ -1315,7 +1311,7 @@ public Boolean exists(Urn urn) { } @Nullable - public RollbackResult deleteAspect(String urn, String aspectName, Map conditions, boolean hardDelete) { + public RollbackResult deleteAspect(String urn, String aspectName, @Nonnull Map conditions, boolean hardDelete) { // Validate pre-conditions before running queries Urn entityUrn; EntitySpec entitySpec; @@ -1323,8 +1319,6 @@ public RollbackResult deleteAspect(String urn, String aspectName, Map getEnvelopedAspects(final S // since nowhere else is using it should be safe for now at least envelopedAspect.setType(AspectType.VERSIONED); envelopedAspect.setValue(aspect); + + try { + if (currAspectEntry.getSystemMetadata() != null) { + final SystemMetadata systemMetadata = RecordUtils.toRecordTemplate(SystemMetadata.class, currAspectEntry.getSystemMetadata()); + envelopedAspect.setSystemMetadata(systemMetadata); + } + } catch (Exception e) { + log.warn("Exception encountered when setting system metadata on enveloped aspect {}. Error: {}", envelopedAspect.getName(), e); + } + envelopedAspect.setCreated(new AuditStamp() .setActor(UrnUtils.getUrn(currAspectEntry.getCreatedBy())) .setTime(currAspectEntry.getCreatedOn().getTime()) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index 7789bc4c3c1eba..87d928871deaa0 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -74,10 +74,15 @@ public static SystemMetadata parseSystemMetadata(String jsonSystemMetadata) { } /** - * Check if entity is removed (removed=true in Status aspect) + * Check if entity is removed (removed=true in Status aspect) and exists */ public static boolean checkIfRemoved(EntityService entityService, Urn entityUrn) { try { + + if (!entityService.exists(entityUrn)) { + return false; + } + EnvelopedAspect statusAspect = entityService.getLatestEnvelopedAspect(entityUrn.getEntityType(), entityUrn, "status"); if (statusAspect == null) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/AspectStorageValidationUtil.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/AspectStorageValidationUtil.java index f19a570d197739..7804aa2067088b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/AspectStorageValidationUtil.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/AspectStorageValidationUtil.java @@ -16,8 +16,8 @@ private AspectStorageValidationUtil() { * @return {@code true} if table exists. */ public static boolean checkTableExists(@Nonnull CqlSession session) { - String query = String.format("SELECT columnfamily_name\n " - + "FROM schema_columnfamilies WHERE keyspace_name='%s';", + String query = String.format("SELECT table_name \n " + + "FROM system_schema.tables where table_name = '%s' allow filtering;", CassandraAspect.TABLE_NAME); ResultSet rs = session.execute(query); return rs.all().size() > 0; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java index 13ce40e8fbb365..a95e8eb62f8516 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java @@ -7,7 +7,6 @@ import com.datastax.oss.driver.api.core.cql.ResultSet; import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; -import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.core.paging.OffsetPager; import com.datastax.oss.driver.api.core.paging.OffsetPager.Page; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; @@ -28,10 +27,6 @@ import com.linkedin.metadata.query.ExtraInfo; import com.linkedin.metadata.query.ExtraInfoArray; import com.linkedin.metadata.query.ListResultMetadata; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.net.URISyntaxException; import java.sql.Timestamp; import java.util.HashMap; @@ -41,13 +36,12 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; -import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.deleteFrom; -import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.insertInto; -import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; -import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; -import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.update; -import static com.linkedin.metadata.Constants.ASPECT_LATEST_VERSION; +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.*; +import static com.linkedin.metadata.Constants.*; @Slf4j public class CassandraAspectDao implements AspectDao, AspectMigrationsDao { @@ -97,10 +91,13 @@ public long countEntities() { SimpleStatement ss = selectFrom(CassandraAspect.TABLE_NAME) .distinct() .column(CassandraAspect.URN_COLUMN) - .countAll() .build(); - return _cqlSession.execute(ss).one().getLong(0); + ResultSet rs = _cqlSession.execute(ss); + // TODO: make sure it doesn't blow up on a large database + // Getting a count of distinct values in a Cassandra query doesn't seem to be feasible, but counting them in the app is dangerous + // The saving grace here is that the only place where this method is used should only run once, what the database is still young + return rs.all().size(); } @Override @@ -110,10 +107,11 @@ public boolean checkIfAspectExists(@Nonnull String aspectName) { .column(CassandraAspect.URN_COLUMN) .whereColumn(CassandraAspect.ASPECT_COLUMN).isEqualTo(literal(aspectName)) .limit(1) + .allowFiltering() .build(); ResultSet rs = _cqlSession.execute(ss); - return rs.all().size() > 0; + return rs.one() != null; } private Map getMaxVersions(@Nonnull final String urn, @Nonnull final Set aspectNames) { @@ -451,7 +449,6 @@ public Iterable listAllUrns(int start, int pageSize) { validateConnection(); SimpleStatement ss = selectFrom(CassandraAspect.TABLE_NAME) .column(CassandraAspect.URN_COLUMN) - .orderBy(CassandraAspect.URN_COLUMN, ClusteringOrder.ASC) .build(); ResultSet rs = _cqlSession.execute(ss); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index c9af475649802e..47fd051ed04548 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -6,9 +6,9 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.AspectMigrationsDao; -import com.linkedin.metadata.entity.ListResult; import com.linkedin.metadata.entity.EntityAspect; import com.linkedin.metadata.entity.EntityAspectIdentifier; +import com.linkedin.metadata.entity.ListResult; import com.linkedin.metadata.query.ExtraInfo; import com.linkedin.metadata.query.ExtraInfoArray; import com.linkedin.metadata.query.ListResultMetadata; @@ -24,12 +24,6 @@ import io.ebean.Transaction; import io.ebean.TxScope; import io.ebean.annotation.TxIsolation; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.persistence.RollbackException; -import javax.persistence.Table; import java.net.URISyntaxException; import java.sql.Timestamp; import java.time.Clock; @@ -41,8 +35,13 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.persistence.RollbackException; +import javax.persistence.Table; +import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.Constants.ASPECT_LATEST_VERSION; +import static com.linkedin.metadata.Constants.*; @Slf4j public class EbeanAspectDao implements AspectDao, AspectMigrationsDao { @@ -233,7 +232,6 @@ public EntityAspect getAspect(@Nonnull final EntityAspectIdentifier key) { } @Override - @Nullable public void deleteAspect(@Nonnull final EntityAspect aspect) { validateConnection(); EbeanAspectV2 ebeanAspect = EbeanAspectV2.fromEntityAspect(aspect); @@ -241,7 +239,6 @@ public void deleteAspect(@Nonnull final EntityAspect aspect) { } @Override - @Nullable public int deleteUrn(@Nonnull final String urn) { validateConnection(); return _server.createQuery(EbeanAspectV2.class).where().eq(EbeanAspectV2.URN_COLUMN, urn).delete(); @@ -310,7 +307,7 @@ private String batchGetSelect( outputParamsToValues.put(aspectArg, aspect); outputParamsToValues.put(versionArg, version); - return String.format("SELECT urn, aspect, version, metadata, createdOn, createdBy, createdFor " + return String.format("SELECT urn, aspect, version, metadata, systemMetadata, createdOn, createdBy, createdFor " + "FROM %s WHERE urn = :%s AND aspect = :%s AND version = :%s", EbeanAspectV2.class.getAnnotation(Table.class).name(), urnArg, aspectArg, versionArg); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java index b010995e0c4da5..906653a9afbd52 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/GraphService.java @@ -94,8 +94,7 @@ RelatedEntitiesResult findRelatedEntities(@Nullable final String sourceType, @No default EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageDirection direction, int offset, int count, int maxHops) { if (maxHops > 1) { - throw new UnsupportedOperationException( - String.format("More than 1 hop is not supported for %s", this.getClass().getSimpleName())); + maxHops = 1; } List edgesToFetch = getLineageRegistry().getLineageRelationships(entityUrn.getEntityType(), direction); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java new file mode 100644 index 00000000000000..9cf08072878dd9 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java @@ -0,0 +1,147 @@ +package com.linkedin.metadata.graph; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.Siblings; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.entity.EntityService; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.Constants.*; + + +@Slf4j +@RequiredArgsConstructor +public class SiblingGraphService { + + private final EntityService _entityService; + private final GraphService _graphService; + + @Nonnull + public EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageDirection direction, int offset, + int count, int maxHops) { + return getLineage(entityUrn, direction, offset, count, maxHops, false); + } + + /** + * Traverse from the entityUrn towards the input direction up to maxHops number of hops + * Abstracts away the concept of relationship types + * + * Unless overridden, it uses the lineage registry to fetch valid edge types and queries for them + */ + @Nonnull + public EntityLineageResult getLineage(@Nonnull Urn entityUrn, @Nonnull LineageDirection direction, int offset, + int count, int maxHops, boolean separateSiblings) { + if (separateSiblings) { + return _graphService.getLineage(entityUrn, direction, offset, count, maxHops); + } + + if (maxHops > 1) { + throw new UnsupportedOperationException( + String.format("More than 1 hop is not supported for %s", this.getClass().getSimpleName())); + } + + EntityLineageResult entityLineage = _graphService.getLineage(entityUrn, direction, offset, count, maxHops); + + Siblings siblingAspectOfEntity = (Siblings) _entityService.getLatestAspect(entityUrn, SIBLINGS_ASPECT_NAME); + + // if you have siblings, we want to fetch their lineage too and merge it in + if (siblingAspectOfEntity != null && siblingAspectOfEntity.hasSiblings()) { + UrnArray siblingUrns = siblingAspectOfEntity.getSiblings(); + Set allSiblingsInGroup = siblingUrns.stream().collect(Collectors.toSet()); + allSiblingsInGroup.add(entityUrn); + + // remove your siblings from your lineage + entityLineage = + filterLineageResultFromSiblings(entityUrn, allSiblingsInGroup, entityLineage, null); + + // Update offset and count to fetch the correct number of edges from the next sibling node + offset = Math.max(0, offset - entityLineage.getTotal()); + count = Math.max(0, count - entityLineage.getRelationships().size()); + + // iterate through each sibling and include their lineage in the bunch + for (Urn siblingUrn : siblingUrns) { + EntityLineageResult nextEntityLineage = filterLineageResultFromSiblings(siblingUrn, allSiblingsInGroup, + _graphService.getLineage(siblingUrn, direction, offset, count, maxHops), entityLineage); + + // Update offset and count to fetch the correct number of edges from the next sibling node + offset = Math.max(0, offset - nextEntityLineage.getTotal()); + count = Math.max(0, count - nextEntityLineage.getCount() - entityLineage.getCount()); + + entityLineage = nextEntityLineage; + }; + } + + return entityLineage; + } + + // takes a lineage result and removes any nodes that are siblings of some other node already in the result + private EntityLineageResult filterLineageResultFromSiblings( + Urn urn, + Set allSiblingsInGroup, + EntityLineageResult entityLineageResult, + EntityLineageResult existingResult + ) { + // 1) remove the source entities siblings from this entity's downstreams + List filteredRelationships = entityLineageResult.getRelationships() + .stream() + .filter(lineageRelationship -> !allSiblingsInGroup.contains(lineageRelationship.getEntity()) + || lineageRelationship.getEntity().equals(urn)) + .collect(Collectors.toList()); + + // 2) combine this entity's lineage with the lineage we've already seen + List combinedResults = Stream.concat( + filteredRelationships.stream(), + existingResult != null ? existingResult.getRelationships().stream() : ImmutableList.of().stream()) + .collect(Collectors.toList()); + + // 3) fetch the siblings of each lineage result + Set combinedResultUrns = combinedResults.stream().map(result -> result.getEntity()).collect(Collectors.toSet()); + + Map> siblingAspects = + _entityService.getLatestAspects(combinedResultUrns, ImmutableSet.of(SIBLINGS_ASPECT_NAME)); + + // 4) if you are not primary & your sibling is in the results, filter yourself out of the return set + filteredRelationships = combinedResults.stream().filter(result -> { + Optional optionalSiblingsAspect = siblingAspects.get(result.getEntity()).stream().filter( + aspect -> aspect instanceof Siblings + ).findAny(); + + if (!optionalSiblingsAspect.isPresent()) { + return true; + } + + + Siblings siblingsAspect = (Siblings) optionalSiblingsAspect.get(); + + if (siblingsAspect.isPrimary()) { + return true; + } + + // if you are not primary and your sibling exists in the result set, filter yourself out + if (siblingsAspect.getSiblings().stream().anyMatch( + sibling -> combinedResultUrns.contains(sibling) + )) { + return false; + } + + return true; + }).collect(Collectors.toList()); + + entityLineageResult.setRelationships(new LineageRelationshipArray(filteredRelationships)); + entityLineageResult.setTotal(entityLineageResult.getTotal() + (existingResult != null ? existingResult.getTotal() : 0)); + entityLineageResult.setCount(filteredRelationships.size()); + return entityLineageResult; + } + +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java index 46bc682f7a8ac2..f7915f46c7fd94 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/EntitySearchService.java @@ -45,6 +45,15 @@ public interface EntitySearchService { */ void deleteDocument(@Nonnull String entityName, @Nonnull String docId); + /** + * Appends a run id to the list for a certain document + * + * @param entityName name of the entity + * @param urn the urn of the user + * @param runId the ID of the run + */ + void appendRunId(@Nonnull String entityName, @Nonnull Urn urn, @Nullable String runId); + /** * Gets a list of documents that match given search request. The results are aggregated and filters are applied to the * search hits and not the aggregation results. diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index 907f6f0632a71e..513d1d4fa4bcf6 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -58,6 +58,7 @@ public class LineageSearchService { * @param direction Direction of the relationship * @param entities list of entities to search (If empty, searches across all entities) * @param input the search input text + * @param maxHops the maximum number of hops away to search for. If null, defaults to 1000 * @param inputFilters the request map with fields and values as filters to be applied to search hits * @param sortCriterion {@link SortCriterion} to be applied to search results * @param from index to start the search from @@ -67,12 +68,13 @@ public class LineageSearchService { @Nonnull @WithSpan public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull LineageDirection direction, - @Nonnull List entities, @Nullable String input, @Nullable Filter inputFilters, + @Nonnull List entities, @Nullable String input, @Nullable Integer maxHops, @Nullable Filter inputFilters, @Nullable SortCriterion sortCriterion, int from, int size) { // Cache multihop result for faster performance EntityLineageResult lineageResult = cache.get(Pair.of(sourceUrn, direction), EntityLineageResult.class); if (lineageResult == null) { - lineageResult = _graphService.getLineage(sourceUrn, direction, 0, MAX_RELATIONSHIPS, 1000); + maxHops = maxHops != null ? maxHops : 1000; + lineageResult = _graphService.getLineage(sourceUrn, direction, 0, MAX_RELATIONSHIPS, maxHops); } // Filter hopped result based on the set of entities to return and inputFilters before sending to search diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java index b8fe920d39f88e..05d314ecb78cf4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java @@ -1,13 +1,11 @@ package com.linkedin.metadata.search; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; -import com.linkedin.metadata.search.cache.AllEntitiesSearchAggregatorCache; +import com.linkedin.metadata.search.cache.CachingAllEntitiesSearchAggregator; import com.linkedin.metadata.search.cache.EntityDocCountCache; -import com.linkedin.metadata.search.cache.EntitySearchServiceCache; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.ranker.SearchRanker; import java.util.List; import java.util.Map; @@ -16,30 +14,24 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.springframework.cache.CacheManager; @Slf4j public class SearchService { - private final EntitySearchService _entitySearchService; - private final AllEntitiesSearchAggregator _aggregator; - private final SearchRanker _searchRanker; - + private final CachingEntitySearchService _cachingEntitySearchService; + private final CachingAllEntitiesSearchAggregator _cachingAllEntitiesSearchAggregator; private final EntityDocCountCache _entityDocCountCache; - private final EntitySearchServiceCache _entitySearchServiceCache; - private final AllEntitiesSearchAggregatorCache _allEntitiesSearchAggregatorCache; + private final SearchRanker _searchRanker; - public SearchService(EntityRegistry entityRegistry, EntitySearchService entitySearchService, - SearchRanker searchRanker, CacheManager cacheManager, int batchSize, boolean enableCache) { - _entitySearchService = entitySearchService; + public SearchService( + EntityDocCountCache entityDocCountCache, + CachingEntitySearchService cachingEntitySearchService, + CachingAllEntitiesSearchAggregator cachingEntitySearchAggregator, + SearchRanker searchRanker) { + _cachingEntitySearchService = cachingEntitySearchService; + _cachingAllEntitiesSearchAggregator = cachingEntitySearchAggregator; _searchRanker = searchRanker; - _aggregator = - new AllEntitiesSearchAggregator(entityRegistry, entitySearchService, searchRanker, cacheManager, batchSize, - enableCache); - _entityDocCountCache = new EntityDocCountCache(entityRegistry, entitySearchService); - _entitySearchServiceCache = new EntitySearchServiceCache(cacheManager, entitySearchService, batchSize, enableCache); - _allEntitiesSearchAggregatorCache = - new AllEntitiesSearchAggregatorCache(cacheManager, _aggregator, batchSize, enableCache); + _entityDocCountCache = entityDocCountCache; } public Map docCountPerEntity(@Nonnull List entityNames) { @@ -65,8 +57,8 @@ public Map docCountPerEntity(@Nonnull List entityNames) { public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags) { SearchResult result = - _entitySearchServiceCache.getSearcher(entityName, input, postFilters, sortCriterion, searchFlags) - .getSearchResults(from, size); + _cachingEntitySearchService.search(entityName, input, postFilters, sortCriterion, from, size, searchFlags); + try { return result.copy().setEntities(new SearchEntityArray(_searchRanker.rank(result.getEntities()))); } catch (Exception e) { @@ -95,7 +87,6 @@ public SearchResult searchAcrossEntities(@Nonnull List entities, @Nonnul log.debug(String.format( "Searching Search documents entities: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s", entities, input, postFilters, sortCriterion, from, size)); - return _allEntitiesSearchAggregatorCache.getSearcher(entities, input, postFilters, sortCriterion, searchFlags) - .getSearchResults(from, size); + return _cachingAllEntitiesSearchAggregator.getSearchResults(entities, input, postFilters, sortCriterion, from, size, searchFlags); } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java b/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java index 554ace2eda18eb..311ca4b0648f6e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/aggregator/AllEntitiesSearchAggregator.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.search.aggregator; import com.codahale.metrics.Timer; +import com.linkedin.data.template.GetMode; import com.linkedin.data.template.LongMap; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.SearchFlags; @@ -14,7 +15,7 @@ import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchResultMetadata; -import com.linkedin.metadata.search.cache.EntitySearchServiceCache; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.cache.EntityDocCountCache; import com.linkedin.metadata.search.ranker.SearchRanker; import com.linkedin.metadata.search.utils.SearchUtils; @@ -27,29 +28,36 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.springframework.cache.CacheManager; import static com.linkedin.metadata.search.utils.FilterUtils.rankFilterGroups; @Slf4j public class AllEntitiesSearchAggregator { + + private static final int DEFAULT_MAX_AGGREGATION_VALUES = 20; + private final EntitySearchService _entitySearchService; private final SearchRanker _searchRanker; private final EntityDocCountCache _entityDocCountCache; - - private final EntitySearchServiceCache _entitySearchServiceCache; - - public AllEntitiesSearchAggregator(EntityRegistry entityRegistry, EntitySearchService entitySearchService, - SearchRanker searchRanker, CacheManager cacheManager, int batchSize, boolean enableCache) { - _entitySearchService = entitySearchService; - _searchRanker = searchRanker; + private final CachingEntitySearchService _cachingEntitySearchService; + private final int _maxAggregationValueCount; + + public AllEntitiesSearchAggregator( + EntityRegistry entityRegistry, + EntitySearchService entitySearchService, + CachingEntitySearchService cachingEntitySearchService, + SearchRanker searchRanker) { + _entitySearchService = Objects.requireNonNull(entitySearchService); + _searchRanker = Objects.requireNonNull(searchRanker); + _cachingEntitySearchService = Objects.requireNonNull(cachingEntitySearchService); _entityDocCountCache = new EntityDocCountCache(entityRegistry, entitySearchService); - _entitySearchServiceCache = new EntitySearchServiceCache(cacheManager, entitySearchService, batchSize, enableCache); + _maxAggregationValueCount = DEFAULT_MAX_AGGREGATION_VALUES; // TODO: Make this externally configurable } @Nonnull @@ -95,10 +103,6 @@ public SearchResult search(@Nonnull List entities, @Nonnull String input Map numResultsPerEntity = searchResults.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getNumEntities().longValue())); - aggregations.put("entity", new AggregationMetadata().setName("entity") - .setDisplayName("Type") - .setAggregations(new LongMap(numResultsPerEntity)) - .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(numResultsPerEntity)))); for (String entity : searchResults.keySet()) { SearchResult result = searchResults.get(entity); @@ -114,10 +118,19 @@ public SearchResult search(@Nonnull List entities, @Nonnull String input }); } + // Trim the aggregations / filters after merging. + Map finalAggregations = trimMergedAggregations(aggregations); + + // Finally, Add a custom Entity aggregation (appears as the first filter) -- this should never be truncated + finalAggregations.put("entity", new AggregationMetadata().setName("entity") + .setDisplayName("Type") + .setAggregations(new LongMap(numResultsPerEntity)) + .setFilterValues(new FilterValueArray(SearchUtil.convertToFilters(numResultsPerEntity)))); + // 4. Rank results across entities List rankedResult = _searchRanker.rank(matchedResults); SearchResultMetadata finalMetadata = - new SearchResultMetadata().setAggregations(new AggregationMetadataArray(rankFilterGroups(aggregations))); + new SearchResultMetadata().setAggregations(new AggregationMetadataArray(rankFilterGroups(finalAggregations))); postProcessTimer.stop(); return new SearchResult().setEntities(new SearchEntityArray(rankedResult)) @@ -143,12 +156,38 @@ private Map getSearchResultsForEachEntity(@Nonnull List new Pair<>(entity, - _entitySearchServiceCache.getSearcher(entity, input, postFilters, sortCriterion, searchFlags) - .getSearchResults(queryFrom, querySize))) + _cachingEntitySearchService.search(entity, input, postFilters, sortCriterion, queryFrom, querySize, searchFlags))) .stream() .filter(pair -> pair.getValue().getNumEntities() > 0) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } return searchResults; } + + /** + * Simply trims the total aggregation values that are returned to the client based on the SearchFlags which are set + */ + private Map trimMergedAggregations(Map aggregations) { + return aggregations.entrySet().stream().map( + entry -> Pair.of(entry.getKey(), new AggregationMetadata() + .setName(entry.getValue().getName()) + .setDisplayName(entry.getValue().getDisplayName(GetMode.NULL)) + .setAggregations(entry.getValue().getAggregations()) + .setFilterValues( + trimFilterValues(entry.getValue().getFilterValues())) + ) + ).collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + } + + /** + * Selects the top N filter values AFTER they've been fully merged. + */ + private FilterValueArray trimFilterValues(FilterValueArray original) { + if (original.size() > _maxAggregationValueCount) { + return new FilterValueArray( + original.subList(0, _maxAggregationValueCount) + ); + } + return original; + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/cache/AllEntitiesSearchAggregatorCache.java b/metadata-io/src/main/java/com/linkedin/metadata/search/cache/CachingAllEntitiesSearchAggregator.java similarity index 73% rename from metadata-io/src/main/java/com/linkedin/metadata/search/cache/AllEntitiesSearchAggregatorCache.java rename to metadata-io/src/main/java/com/linkedin/metadata/search/cache/CachingAllEntitiesSearchAggregator.java index 27ca66b7dadb82..c8286d7ca99c2c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/cache/AllEntitiesSearchAggregatorCache.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/cache/CachingAllEntitiesSearchAggregator.java @@ -3,6 +3,7 @@ import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; import java.util.List; import javax.annotation.Nonnull; @@ -13,7 +14,7 @@ @RequiredArgsConstructor -public class AllEntitiesSearchAggregatorCache { +public class CachingAllEntitiesSearchAggregator { private static final String ALL_ENTITIES_SEARCH_AGGREGATOR_CACHE_NAME = "allEntitiesSearchAggregator"; private final CacheManager cacheManager; @@ -21,11 +22,11 @@ public class AllEntitiesSearchAggregatorCache { private final int batchSize; private final boolean enableCache; - public CacheableSearcher getSearcher(List entities, @Nonnull String input, @Nullable Filter postFilters, - @Nullable SortCriterion sortCriterion, @Nullable SearchFlags searchFlags) { + public SearchResult getSearchResults(List entities, @Nonnull String input, @Nullable Filter postFilters, + @Nullable SortCriterion sortCriterion, int from, int size, @Nullable SearchFlags searchFlags) { return new CacheableSearcher<>(cacheManager.getCache(ALL_ENTITIES_SEARCH_AGGREGATOR_CACHE_NAME), batchSize, querySize -> aggregator.search(entities, input, postFilters, sortCriterion, querySize.getFrom(), querySize.getSize(), searchFlags), - querySize -> Quintet.with(entities, input, postFilters, sortCriterion, querySize), searchFlags, enableCache); + querySize -> Quintet.with(entities, input, postFilters, sortCriterion, querySize), searchFlags, enableCache).getSearchResults(from, size); } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntitySearchServiceCache.java b/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntitySearchServiceCache.java deleted file mode 100644 index cc5d8c91d4d1b4..00000000000000 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/cache/EntitySearchServiceCache.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.linkedin.metadata.search.cache; - -import com.linkedin.metadata.query.SearchFlags; -import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.search.EntitySearchService; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import lombok.RequiredArgsConstructor; -import org.javatuples.Quintet; -import org.springframework.cache.CacheManager; - - -@RequiredArgsConstructor -public class EntitySearchServiceCache { - private static final String ENTITY_SEARCH_SERVICE_CACHE_NAME = "entitySearchService"; - - private final CacheManager cacheManager; - private final EntitySearchService entitySearchService; - private final int batchSize; - private final boolean enableCache; - - public CacheableSearcher getSearcher(@Nonnull String entityName, @Nonnull String input, - @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, @Nullable SearchFlags searchFlags) { - return new CacheableSearcher<>(cacheManager.getCache(ENTITY_SEARCH_SERVICE_CACHE_NAME), batchSize, - querySize -> entitySearchService.search(entityName, input, postFilters, sortCriterion, querySize.getFrom(), - querySize.getSize()), querySize -> Quintet.with(entityName, input, postFilters, sortCriterion, querySize), - searchFlags, enableCache); - } -} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java new file mode 100644 index 00000000000000..a2d2b8857ae489 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java @@ -0,0 +1,175 @@ +package com.linkedin.metadata.search.client; + +import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.cache.CacheableSearcher; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import org.javatuples.Quintet; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; + + +@RequiredArgsConstructor +public class CachingEntitySearchService { + private static final String ENTITY_SEARCH_SERVICE_SEARCH_CACHE_NAME = "entitySearchServiceSearch"; + private static final String ENTITY_SEARCH_SERVICE_AUTOCOMPLETE_CACHE_NAME = "entitySearchServiceAutoComplete"; + + private final CacheManager cacheManager; + private final EntitySearchService entitySearchService; // This is a shared component, also used in search aggregation + private final int batchSize; + private final boolean enableCache; + + /** + * Retrieves cached search results. If the query has been cached, this will return quickly. If not, a full + * search request will be made. + * + * @param entityName the name of the entity to search + * @param query the search query + * @param filters the filters to include + * @param sortCriterion the sort criterion + * @param from the start offset + * @param size the count + * @param flags additional search flags + * + * @return a {@link SearchResult} containing the requested batch of search results + */ + public SearchResult search( + @Nonnull String entityName, + @Nonnull String query, + @Nullable Filter filters, + @Nullable SortCriterion sortCriterion, + int from, + int size, + @Nullable SearchFlags flags) { + return getCachedSearchResults(entityName, query, filters, sortCriterion, from, size, flags); + } + + /** + * Retrieves cached auto complete results + * + * @param entityName the name of the entity to search + * @param input the input query + * @param filters the filters to include + * @param limit the max number of results to return + * @param flags additional search flags + * + * @return a {@link SearchResult} containing the requested batch of search results + */ + public AutoCompleteResult autoComplete( + @Nonnull String entityName, + @Nonnull String input, + @Nullable String field, + @Nullable Filter filters, + int limit, + @Nullable SearchFlags flags) { + return getCachedAutoCompleteResults(entityName, input, field, filters, limit, flags); + } + + /** + * Get search results corresponding to the input "from" and "size" + * It goes through batches, starting from the beginning, until we get enough results to return + * This let's us have batches that return a variable number of results (we have no idea which batch the "from" "size" page corresponds to) + */ + public SearchResult getCachedSearchResults( + @Nonnull String entityName, + @Nonnull String query, + @Nullable Filter filters, + @Nullable SortCriterion sortCriterion, + int from, + int size, + @Nullable SearchFlags flags) { + return new CacheableSearcher<>( + cacheManager.getCache(ENTITY_SEARCH_SERVICE_SEARCH_CACHE_NAME), + batchSize, + querySize -> getRawSearchResults(entityName, query, filters, sortCriterion, querySize.getFrom(), querySize.getSize()), + querySize -> Quintet.with(entityName, query, filters, sortCriterion, querySize), flags, enableCache).getSearchResults(from, size); + } + + + /** + * Returns cached auto-complete results. + */ + public AutoCompleteResult getCachedAutoCompleteResults( + @Nonnull String entityName, + @Nonnull String input, + @Nullable String field, + @Nullable Filter filters, + int limit, + @Nullable SearchFlags flags) { + Cache cache = cacheManager.getCache(ENTITY_SEARCH_SERVICE_AUTOCOMPLETE_CACHE_NAME); + AutoCompleteResult result; + if (enableCache(flags)) { + Object cacheKey = Quintet.with(entityName, input, field, filters, limit); + result = cache.get(cacheKey, AutoCompleteResult.class); + if (result == null) { + result = getRawAutoCompleteResults( + entityName, + input, + field, + filters, + limit + ); + cache.put(cacheKey, result); + } + } else { + result = getRawAutoCompleteResults( + entityName, + input, + field, + filters, + limit + ); + } + return result; + } + + /** + * Executes the expensive search query using the {@link EntitySearchService} + */ + private SearchResult getRawSearchResults( + final String entityName, + final String input, + final Filter filters, + final SortCriterion sortCriterion, + final int start, + final int count) { + return entitySearchService.search( + entityName, + input, + filters, + sortCriterion, + start, + count); + } + + /** + * Executes the expensive autocomplete query using the {@link EntitySearchService} + */ + private AutoCompleteResult getRawAutoCompleteResults( + final String entityName, + final String input, + final String field, + final Filter filters, + final int limit) { + return entitySearchService.autoComplete( + entityName, + input, + field, + filters, + limit); + } + + /** + * Returns true if the cache should be used or skipped when fetching search results + */ + private boolean enableCache(final SearchFlags searchFlags) { + return enableCache && (searchFlags == null || !searchFlags.isSkipCache()); + } + +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index ae40f9af23e4d2..4e0eb2489dbee3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -12,8 +12,10 @@ import com.linkedin.metadata.search.elasticsearch.query.ESSearchDAO; import com.linkedin.metadata.search.elasticsearch.update.ESWriteDAO; import com.linkedin.metadata.search.utils.ESUtils; +import com.linkedin.metadata.search.utils.SearchUtils; import java.util.List; import java.util.Map; +import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; @@ -24,6 +26,7 @@ @RequiredArgsConstructor public class ElasticSearchService implements EntitySearchService { + private static final int MAX_RUN_IDS_INDEXED = 25; // Save the previous 25 run ids in the index. private final EntityIndexBuilders indexBuilders; private final ESSearchDAO esSearchDAO; private final ESBrowseDAO esBrowseDAO; @@ -57,6 +60,33 @@ public void deleteDocument(@Nonnull String entityName, @Nonnull String docId) { esWriteDAO.deleteDocument(entityName, docId); } + @Override + public void appendRunId(@Nonnull String entityName, @Nonnull Urn urn, @Nullable String runId) { + final Optional maybeDocId = SearchUtils.getDocId(urn); + if (!maybeDocId.isPresent()) { + log.warn(String.format("Failed to append run id, could not generate a doc id for urn %s", urn)); + return; + } + final String docId = maybeDocId.get(); + log.debug(String.format("Appending run id for entityName: %s, docId: %s", entityName, docId)); + esWriteDAO.applyScriptUpdate(entityName, docId, + /* + Script used to apply updates to the runId field of the index. + This script saves the past N run ids which touched a particular URN in the search index. + It only adds a new run id if it is not already stored inside the list. (List is unique AND ordered) + */ + String.format( + "if (ctx._source.containsKey('runId')) { " + + "if (!ctx._source.runId.contains('%s')) { " + + "ctx._source.runId.add('%s'); " + + "if (ctx._source.runId.length > %s) { ctx._source.runId.remove(0) } } " + + "} else { ctx._source.runId = ['%s'] }", + runId, + runId, + MAX_RUN_IDS_INDEXED, + runId)); + } + @Nonnull @Override public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index 9ad3374855ee07..b8c07ce7bb5f6e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -19,7 +19,11 @@ private MappingsBuilder() { public static Map getMappings(@Nonnull final EntitySpec entitySpec) { Map mappings = new HashMap<>(); + + // Fixed fields mappings.put("urn", getMappingsForUrn()); + mappings.put("runId", getMappingsForRunId()); + entitySpec.getSearchableFieldSpecs() .forEach(searchableFieldSpec -> mappings.putAll(getMappingsForField(searchableFieldSpec))); entitySpec.getSearchScoreFieldSpecs() @@ -31,6 +35,10 @@ private static Map getMappingsForUrn() { return ImmutableMap.builder().put("type", "keyword").build(); } + private static Map getMappingsForRunId() { + return ImmutableMap.builder().put("type", "keyword").build(); + } + private static Map getMappingsForField(@Nonnull final SearchableFieldSpec searchableFieldSpec) { FieldType fieldType = searchableFieldSpec.getSearchableAnnotation().getFieldType(); boolean addToFilters = searchableFieldSpec.getSearchableAnnotation().isAddToFilters(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 187b8398bd3a15..ae0fd94af04e56 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -36,6 +36,9 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -58,15 +61,28 @@ public class SearchRequestHandler { private static final Map REQUEST_HANDLER_BY_ENTITY_NAME = new ConcurrentHashMap<>(); + private static final String REMOVED = "removed"; + private static final int DEFAULT_MAX_TERM_BUCKET_SIZE = 20; private final EntitySpec _entitySpec; private final Set _facetFields; private final Set _defaultQueryFieldNames; private final Map _filtersToDisplayName; - private final int _maxTermBucketSize = 100; - private static final String REMOVED = "removed"; + private final Configs _configs; + + + @Data + @AllArgsConstructor + @Getter + public static class Configs { + private final int maxTermBucketSize; + } private SearchRequestHandler(@Nonnull EntitySpec entitySpec) { + this(entitySpec, new Configs(DEFAULT_MAX_TERM_BUCKET_SIZE)); + } + + private SearchRequestHandler(@Nonnull EntitySpec entitySpec, @Nonnull Configs configs) { _entitySpec = entitySpec; _facetFields = getFacetFields(); _defaultQueryFieldNames = getDefaultQueryFieldNames(); @@ -75,12 +91,17 @@ private SearchRequestHandler(@Nonnull EntitySpec entitySpec) { .filter(spec -> spec.getSearchableAnnotation().isAddToFilters()) .collect(Collectors.toMap(spec -> spec.getSearchableAnnotation().getFieldName(), spec -> spec.getSearchableAnnotation().getFilterName())); + _configs = configs; } public static SearchRequestHandler getBuilder(@Nonnull EntitySpec entitySpec) { return REQUEST_HANDLER_BY_ENTITY_NAME.computeIfAbsent(entitySpec, k -> new SearchRequestHandler(entitySpec)); } + public static SearchRequestHandler getBuilder(@Nonnull EntitySpec entitySpec, @Nonnull Configs configs) { + return REQUEST_HANDLER_BY_ENTITY_NAME.computeIfAbsent(entitySpec, k -> new SearchRequestHandler(entitySpec, configs)); + } + private Set getFacetFields() { return _entitySpec.getSearchableFieldSpecs() .stream() @@ -205,7 +226,7 @@ private List getAggregations() { for (String facet : _facetFields) { // All facet fields must have subField keyword AggregationBuilder aggBuilder = - AggregationBuilders.terms(facet).field(facet + ESUtils.KEYWORD_SUFFIX).size(_maxTermBucketSize); + AggregationBuilders.terms(facet).field(facet + ESUtils.KEYWORD_SUFFIX).size(_configs.getMaxTermBucketSize()); aggregationBuilders.add(aggBuilder); } return aggregationBuilders; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java index d557f3227e58d0..cda77412baebac 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/update/ESWriteDAO.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.script.Script; @Slf4j @@ -54,6 +55,14 @@ public void deleteDocument(@Nonnull String entityName, @Nonnull String docId) { bulkProcessor.add(new DeleteRequest(indexName).id(docId)); } + /** + * Applies a script to a particular document + */ + public void applyScriptUpdate(@Nonnull String entityName, @Nonnull String docId, @Nonnull String script) { + final String indexName = indexConvention.getIndexName(entityRegistry.getEntitySpec(entityName)); + bulkProcessor.add(new UpdateRequest(indexName, docId).script(new Script(script))); + } + /** * Clear all documents in all the indices */ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index 5a26131b45594a..4c527b7cef7f26 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -47,7 +47,10 @@ public Optional transformSnapshot(final RecordTemplate snapshot, final E return Optional.of(searchDocument.toString()); } - public Optional transformAspect(final Urn urn, final RecordTemplate aspect, final AspectSpec aspectSpec, + public Optional transformAspect( + final Urn urn, + final RecordTemplate aspect, + final AspectSpec aspectSpec, final Boolean forDelete) { final Map> extractedSearchableFields = FieldExtractor.extractFields(aspect, aspectSpec.getSearchableFieldSpecs()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 658c6945b3be82..bc3066628abcd7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -60,7 +60,8 @@ public static BoolQueryBuilder buildFilterQuery(@Nullable Filter filter) { log.warn("Received query Filter with a deprecated field 'criteria'. Use 'or' instead."); final BoolQueryBuilder andQueryBuilder = new BoolQueryBuilder(); filter.getCriteria().forEach(criterion -> { - if (!criterion.getValue().trim().isEmpty() || criterion.hasValues()) { + if (!criterion.getValue().trim().isEmpty() || criterion.hasValues() + || criterion.getCondition() == Condition.IS_NULL) { andQueryBuilder.must(getQueryBuilderFromCriterion(criterion)); } }); @@ -73,7 +74,8 @@ public static BoolQueryBuilder buildFilterQuery(@Nullable Filter filter) { public static BoolQueryBuilder buildConjunctiveFilterQuery(@Nonnull ConjunctiveCriterion conjunctiveCriterion) { final BoolQueryBuilder andQueryBuilder = new BoolQueryBuilder(); conjunctiveCriterion.getAnd().forEach(criterion -> { - if (!criterion.getValue().trim().isEmpty() || criterion.hasValues()) { + if (!criterion.getValue().trim().isEmpty() || criterion.hasValues() + || criterion.getCondition() == Condition.IS_NULL) { andQueryBuilder.must(getQueryBuilderFromCriterion(criterion)); } }); @@ -120,6 +122,8 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull Criterion crite Arrays.stream(criterion.getValue().trim().split("\\s*,\\s*")) .forEach(elem -> filters.should(QueryBuilders.matchQuery(criterion.getField(), elem))); return filters; + } else if (condition == Condition.IS_NULL) { + return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(criterion.getField())); } else if (condition == Condition.GREATER_THAN) { return QueryBuilders.rangeQuery(criterion.getField()).gt(criterion.getValue().trim()); } else if (condition == Condition.GREATER_THAN_OR_EQUAL_TO) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java new file mode 100644 index 00000000000000..e4b23aba7b92c8 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java @@ -0,0 +1,69 @@ +package com.linkedin.metadata; + +import com.linkedin.chart.ChartInfo; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.ChangeAuditStamps; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.key.CorpUserKey; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.PegasusUtils; +import com.linkedin.mxe.SystemMetadata; +import javax.annotation.Nonnull; + + +public class AspectGenerationUtils { + + private AspectGenerationUtils() { + } + + @Nonnull + public static AuditStamp createAuditStamp() { + return new AuditStamp().setTime(123L).setActor(UrnUtils.getUrn("urn:li:corpuser:tester")); + } + + @Nonnull + public static SystemMetadata createSystemMetadata() { + return createSystemMetadata(1625792689, "run-123"); + } + + @Nonnull + public static SystemMetadata createSystemMetadata(long lastObserved, @Nonnull String runId) { + SystemMetadata metadata = new SystemMetadata(); + metadata.setLastObserved(lastObserved); + metadata.setRunId(runId); + return metadata; + } + + @Nonnull + public static CorpUserKey createCorpUserKey(Urn urn) { + return (CorpUserKey) EntityKeyUtils.convertUrnToEntityKey(urn, new CorpUserKey().schema()); + } + + @Nonnull + public static CorpUserInfo createCorpUserInfo(@Nonnull String email) { + CorpUserInfo corpUserInfo = new CorpUserInfo(); + corpUserInfo.setEmail(email); + corpUserInfo.setActive(true); + return corpUserInfo; + } + + @Nonnull + public static ChartInfo createChartInfo(@Nonnull String title, @Nonnull String description) { + ChartInfo chartInfo = new ChartInfo(); + chartInfo.setTitle(title); + chartInfo.setDescription(description); + ChangeAuditStamps lastModified = new ChangeAuditStamps(); + lastModified.setCreated(createAuditStamp()); + lastModified.setLastModified(createAuditStamp()); + chartInfo.setLastModified(lastModified); + return chartInfo; + } + + @Nonnull + public static String getAspectName(RecordTemplate record) { + return PegasusUtils.getAspectNameFromSchema(record.schema()); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java new file mode 100644 index 00000000000000..2361bcc22780ad --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java @@ -0,0 +1,75 @@ +package com.linkedin.metadata; + +import com.linkedin.chart.ChartInfo; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.key.CorpUserKey; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nonnull; + + +public class AspectIngestionUtils { + + private AspectIngestionUtils() { + } + + @Nonnull + public static Map ingestCorpUserKeyAspects(EntityService entityService, int aspectCount) { + return ingestCorpUserKeyAspects(entityService, aspectCount, 0); + } + + @Nonnull + public static Map ingestCorpUserKeyAspects(EntityService entityService, int aspectCount, int startIndex) { + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserKey()); + Map aspects = new HashMap<>(); + for (int i = startIndex; i < startIndex + aspectCount; i++) { + Urn urn = UrnUtils.getUrn(String.format("urn:li:corpuser:tester%d", i)); + CorpUserKey aspect = AspectGenerationUtils.createCorpUserKey(urn); + aspects.put(urn, aspect); + entityService.ingestAspect(urn, aspectName, aspect, AspectGenerationUtils.createAuditStamp(), AspectGenerationUtils.createSystemMetadata()); + } + return aspects; + } + + @Nonnull + public static Map ingestCorpUserInfoAspects(@Nonnull final EntityService entityService, int aspectCount) { + return ingestCorpUserInfoAspects(entityService, aspectCount, 0); + } + + @Nonnull + public static Map ingestCorpUserInfoAspects(@Nonnull final EntityService entityService, int aspectCount, int startIndex) { + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); + Map aspects = new HashMap<>(); + for (int i = startIndex; i < startIndex + aspectCount; i++) { + Urn urn = UrnUtils.getUrn(String.format("urn:li:corpuser:tester%d", i)); + String email = String.format("email%d@test.com", i); + CorpUserInfo aspect = AspectGenerationUtils.createCorpUserInfo(email); + aspects.put(urn, aspect); + entityService.ingestAspect(urn, aspectName, aspect, AspectGenerationUtils.createAuditStamp(), AspectGenerationUtils.createSystemMetadata()); + } + return aspects; + } + + @Nonnull + public static Map ingestChartInfoAspects(@Nonnull final EntityService entityService, int aspectCount) { + return ingestChartInfoAspects(entityService, aspectCount, 0); + } + + @Nonnull + public static Map ingestChartInfoAspects(@Nonnull final EntityService entityService, int aspectCount, int startIndex) { + String aspectName = AspectGenerationUtils.getAspectName(new ChartInfo()); + Map aspects = new HashMap<>(); + for (int i = startIndex; i < startIndex + aspectCount; i++) { + Urn urn = UrnUtils.getUrn(String.format("urn:li:chart:(looker,test%d)", i)); + String title = String.format("Test Title %d", i); + String description = String.format("Test description %d", i); + ChartInfo aspect = AspectGenerationUtils.createChartInfo(title, description); + aspects.put(urn, aspect); + entityService.ingestAspect(urn, aspectName, aspect, AspectGenerationUtils.createAuditStamp(), AspectGenerationUtils.createSystemMetadata()); + } + return aspects; + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java new file mode 100644 index 00000000000000..e780e1375a445a --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/AspectMigrationsDaoTest.java @@ -0,0 +1,98 @@ +package com.linkedin.metadata.entity; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.AspectIngestionUtils; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.key.CorpUserKey; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.models.registry.MergedEntityRegistry; +import com.linkedin.metadata.snapshot.Snapshot; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.*; + + +abstract public class AspectMigrationsDaoTest { + + protected T _migrationsDao; + + protected final EntityRegistry _snapshotEntityRegistry; + protected final EntityRegistry _configEntityRegistry; + protected final EntityRegistry _testEntityRegistry; + protected EventProducer _mockProducer; + + protected EntityService _entityService; + protected RetentionService _retentionService; + + protected AspectMigrationsDaoTest() throws EntityRegistryException { + _snapshotEntityRegistry = new TestEntityRegistry(); + _configEntityRegistry = new ConfigEntityRegistry(Snapshot.class.getClassLoader().getResourceAsStream("entity-registry.yml")); + _testEntityRegistry = new MergedEntityRegistry(_snapshotEntityRegistry).apply(_configEntityRegistry); + } + + @Test + public void testListAllUrns() throws AssertionError { + final int totalAspects = 30; + final int pageSize = 25; + final int lastPageSize = 5; + Map ingestedAspects = AspectIngestionUtils.ingestCorpUserKeyAspects(_entityService, totalAspects); + List ingestedUrns = ingestedAspects.keySet().stream().map(Urn::toString).collect(Collectors.toList()); + List seenUrns = new ArrayList<>(); + + Iterable page1 = _migrationsDao.listAllUrns(0, pageSize); + List page1Urns = ImmutableList.copyOf(page1); + + // validate first page + assertEquals(page1Urns.size(), pageSize); + for (String urn : page1Urns) { + assertNotNull(UrnUtils.getUrn(urn)); + seenUrns.add(urn); + } + + Iterable page2 = _migrationsDao.listAllUrns(pageSize, pageSize); + List page2Urns = ImmutableList.copyOf(page2); + + // validate last page + assertEquals(page2Urns.size(), lastPageSize); + for (String urn : page2Urns) { + assertNotNull(UrnUtils.getUrn(urn)); + seenUrns.add(urn); + } + + // validate all ingested URNs were returned exactly once + for (String urn : ingestedUrns) { + assertEquals(seenUrns.stream().filter(u -> u.equals(urn)).count(), 1); + } + } + + @Test + public void testCountEntities() throws AssertionError { + AspectIngestionUtils.ingestCorpUserInfoAspects(_entityService, 11); + AspectIngestionUtils.ingestChartInfoAspects(_entityService, 22); + final int expected = 33; + + long actual = _migrationsDao.countEntities(); + + assertEquals(actual, expected); + } + + @Test + public void testCheckIfAspectExists() throws AssertionError { + boolean actual = _migrationsDao.checkIfAspectExists(CORP_USER_INFO_ASPECT_NAME); + assertFalse(actual); + + AspectIngestionUtils.ingestCorpUserInfoAspects(_entityService, 1); + + actual = _migrationsDao.checkIfAspectExists(CORP_USER_INFO_ASPECT_NAME); + assertTrue(actual); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java new file mode 100644 index 00000000000000..b3d2679fc3be58 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraAspectMigrationsDaoTest.java @@ -0,0 +1,63 @@ +package com.linkedin.metadata.entity; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.linkedin.metadata.CassandraTestUtils; +import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; +import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.models.registry.EntityRegistryException; +import org.testcontainers.containers.CassandraContainer; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static org.mockito.Mockito.*; + + +public class CassandraAspectMigrationsDaoTest extends AspectMigrationsDaoTest { + + private CassandraContainer _cassandraContainer; + + public CassandraAspectMigrationsDaoTest() throws EntityRegistryException { + } + + @BeforeClass + public void setupContainer() { + _cassandraContainer = CassandraTestUtils.setupContainer(); + } + + @AfterClass + public void tearDown() { + _cassandraContainer.stop(); + } + + @BeforeMethod + public void setupTest() { + CassandraTestUtils.purgeData(_cassandraContainer); + configureComponents(); + } + + private void configureComponents() { + CqlSession session = CassandraTestUtils.createTestSession(_cassandraContainer); + CassandraAspectDao dao = new CassandraAspectDao(session); + dao.setConnectionValidated(true); + _mockProducer = mock(EventProducer.class); + _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry); + _retentionService = new CassandraRetentionService(_entityService, session, 1000); + _entityService.setRetentionService(_retentionService); + + _migrationsDao = dao; + } + + /** + * Ideally, all tests would be in the base class, so they're reused between all implementations. + * When that's the case - test runner will ignore this class (and its base!) so we keep this dummy test + * to make sure this class will always be discovered. + */ + @Test + public void obligatoryTest() throws AssertionError { + Assert.assertTrue(true); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java index 5dd17567589340..ab716ed7b0f9ad 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/CassandraEntityServiceTest.java @@ -2,18 +2,23 @@ import com.datastax.oss.driver.api.core.CqlSession; import com.linkedin.common.urn.Urn; -import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.AspectGenerationUtils; +import com.linkedin.metadata.AspectIngestionUtils; import com.linkedin.metadata.CassandraTestUtils; import com.linkedin.metadata.entity.cassandra.CassandraAspectDao; import com.linkedin.metadata.entity.cassandra.CassandraRetentionService; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.query.ExtraInfo; import com.linkedin.metadata.query.ListUrnsResult; -import com.linkedin.metadata.utils.PegasusUtils; -import com.linkedin.mxe.SystemMetadata; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.testcontainers.containers.CassandraContainer; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -21,9 +26,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; /** * A class that knows how to configure {@link EntityServiceTest} to run integration tests against a Cassandra database. @@ -71,110 +75,105 @@ private void configureComponents() { * to make sure this class will always be discovered. */ @Test - public void obligatoryTest() throws Exception { + public void obligatoryTest() throws AssertionError { Assert.assertTrue(true); } @Override @Test - public void testIngestListLatestAspects() throws Exception { + public void testIngestListLatestAspects() throws AssertionError { // TODO: If you're modifying this test - match your changes in sibling implementations. // TODO: Move this test into the base class, // If you can find a way for Cassandra and relational databases to share result ordering rules. - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:test2"); - Urn entityUrn3 = Urn.createFromString("urn:li:corpuser:test3"); - - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); - - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); - - // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - - // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email2@test.com"); - _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); - - // Ingest CorpUserInfo Aspect #3 - CorpUserInfo writeAspect3 = createCorpUserInfo("email3@test.com"); - _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); - - // List aspects - ListResult batch1 = _entityService.listLatestAspects(entityUrn1.getEntityType(), aspectName, 0, 2); - - assertEquals(batch1.getNextStart(), 2); - assertEquals(batch1.getPageSize(), 2); - assertEquals(batch1.getTotalCount(), 3); - assertEquals(batch1.getTotalPageCount(), 2); - assertEquals(batch1.getValues().size(), 2); - assertTrue(DataTemplateUtil.areEqual(writeAspect1, batch1.getValues().get(0))); - assertTrue(DataTemplateUtil.areEqual(writeAspect3, batch1.getValues().get(1))); - - ListResult batch2 = _entityService.listLatestAspects(entityUrn1.getEntityType(), aspectName, 2, 2); - assertEquals(batch2.getValues().size(), 1); - assertTrue(DataTemplateUtil.areEqual(writeAspect2, batch2.getValues().get(0))); + final int totalEntities = 100; + final int pageSize = 30; + final int expectedTotalPages = 4; + final int expectedEntitiesInLastPage = 10; + + Map writtenAspects = AspectIngestionUtils.ingestCorpUserInfoAspects(_entityService, totalEntities); + Set writtenUrns = writtenAspects.keySet(); + String entity = writtenUrns.stream().findFirst().get().getEntityType(); + String aspect = AspectGenerationUtils.getAspectName(new CorpUserInfo()); + + List readUrns = new ArrayList<>(); + for (int pageNo = 0; pageNo < expectedTotalPages; pageNo++) { + boolean isLastPage = pageNo == expectedTotalPages - 1; + int pageStart = pageNo * pageSize; + int expectedEntityCount = isLastPage ? expectedEntitiesInLastPage : pageSize; + int expectedNextStart = isLastPage ? -1 : pageStart + pageSize; + + ListResult page = _entityService.listLatestAspects(entity, aspect, pageStart, pageSize); + + // Check paging metadata works as expected + assertEquals(page.getNextStart(), expectedNextStart); + assertEquals(page.getPageSize(), pageSize); + assertEquals(page.getTotalCount(), totalEntities); + assertEquals(page.getTotalPageCount(), expectedTotalPages); + assertEquals(page.getValues().size(), expectedEntityCount); + + // Remember all URNs we've seen returned for later assertions + readUrns.addAll(page.getMetadata().getExtraInfos().stream().map(ExtraInfo::getUrn).collect(Collectors.toList())); + } + assertEquals(readUrns.size(), writtenUrns.size()); + + // Check that all URNs we've created were seen in some page or other (also check that none were seen more than once) + // We can't be strict on exact order of items in the responses because Cassandra query limitations get in the way here. + for (Urn wUrn : writtenUrns) { + long matchingUrnCount = readUrns.stream().filter(rUrn -> rUrn.toString().equals(wUrn.toString())).count(); + assertEquals(matchingUrnCount, 1L, String.format("Each URN should appear exactly once. %s appeared %d times.", wUrn, matchingUrnCount)); + } } @Override @Test - public void testIngestListUrns() throws Exception { + public void testIngestListUrns() throws AssertionError { // TODO: If you're modifying this test - match your changes in sibling implementations. // TODO: Move this test into the base class, // If you can find a way for Cassandra and relational databases to share result ordering rules. - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:test2"); - Urn entityUrn3 = Urn.createFromString("urn:li:corpuser:test3"); - - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); - - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserKey().schema()); - - // Ingest CorpUserInfo Aspect #1 - RecordTemplate writeAspect1 = createCorpUserKey(entityUrn1); - _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - - // Ingest CorpUserInfo Aspect #2 - RecordTemplate writeAspect2 = createCorpUserKey(entityUrn2); - _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); - - // Ingest CorpUserInfo Aspect #3 - RecordTemplate writeAspect3 = createCorpUserKey(entityUrn3); - _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); - - // List aspects urns - ListUrnsResult batch1 = _entityService.listUrns(entityUrn1.getEntityType(), 0, 2); - - assertEquals((int) batch1.getStart(), 0); - assertEquals((int) batch1.getCount(), 2); - assertEquals((int) batch1.getTotal(), 3); - assertEquals(batch1.getEntities().size(), 2); - assertEquals(entityUrn1.toString(), batch1.getEntities().get(0).toString()); - assertEquals(entityUrn3.toString(), batch1.getEntities().get(1).toString()); - - ListUrnsResult batch2 = _entityService.listUrns(entityUrn1.getEntityType(), 2, 2); - - assertEquals((int) batch2.getStart(), 2); - assertEquals((int) batch2.getCount(), 1); - assertEquals((int) batch2.getTotal(), 3); - assertEquals(batch2.getEntities().size(), 1); - assertEquals(entityUrn2.toString(), batch2.getEntities().get(0).toString()); + final int totalEntities = 100; + final int pageSize = 30; + final int expectedTotalPages = 4; + final int expectedEntitiesInLastPage = 10; + + Map writtenAspects = AspectIngestionUtils.ingestCorpUserKeyAspects(_entityService, totalEntities); + Set writtenUrns = writtenAspects.keySet(); + String entity = writtenUrns.stream().findFirst().get().getEntityType(); + + List readUrns = new ArrayList<>(); + for (int pageNo = 0; pageNo < expectedTotalPages; pageNo++) { + boolean isLastPage = pageNo == expectedTotalPages - 1; + int pageStart = pageNo * pageSize; + int expectedEntityCount = isLastPage ? expectedEntitiesInLastPage : pageSize; + + ListUrnsResult page = _entityService.listUrns(entity, pageStart, pageSize); + + // Check paging metadata works as expected + assertEquals(page.getStart().intValue(), pageStart); + assertEquals(page.getTotal().intValue(), totalEntities); + assertEquals(page.getEntities().size(), expectedEntityCount); + + // Remember all URNs we've seen returned for later assertions + readUrns.addAll(page.getEntities()); + } + assertEquals(readUrns.size(), writtenUrns.size()); + + // Check that all URNs we've created were seen in some page or other (also check that none were seen more than once) + // We can't be strict on exact order of items in the responses because Cassandra query limitations get in the way here. + for (Urn wUrn : writtenUrns) { + long matchingUrnCount = readUrns.stream().filter(rUrn -> rUrn.toString().equals(wUrn.toString())).count(); + assertEquals(matchingUrnCount, 1L, String.format("Each URN should appear exactly once. %s appeared %d times.", wUrn, matchingUrnCount)); + } } @Override @Test - public void testNestedTransactions() throws Exception { + public void testNestedTransactions() { // Doesn't look like Cassandra can support nested transactions (or nested batching). } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java index 13accc02594100..67c9bd0a9e0147 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/DeleteEntityUtilsTest.java @@ -73,18 +73,12 @@ public void testNonOptionalFieldRemoval() { + "}"); final DataSchema schema = pdlSchemaParser.lookupName("simple_record"); - final Aspect updatedAspect = DeleteEntityUtils.getAspectWithReferenceRemoved("hello", aspect, schema, - new PathSpec("key_a")); - - assertTrue(updatedAspect.data().containsKey("key_a")); - assertEquals("hello", updatedAspect.data().get("key_a")); - assertTrue(updatedAspect.data().containsKey("key_b")); - assertEquals("world", updatedAspect.data().get("key_b")); - assertEquals(aspect, updatedAspect); + assertNull(DeleteEntityUtils.getAspectWithReferenceRemoved("hello", aspect, schema, + new PathSpec("key_a"))); } /** - * Tests that Aspect Processor does not delete a non-optional value from a record referenced by another record. + * Tests that Aspect Processor deletes a required value from a record referenced by another record. */ @Test public void testNestedFieldRemoval() { @@ -98,15 +92,14 @@ public void testNestedFieldRemoval() { + "}"); pdlSchemaParser.parse("record complex_record {\n" - + "key_c: simple_record\n" + + "key_c: optional simple_record\n" + "}"); final DataSchema schema = pdlSchemaParser.lookupName("complex_record"); final Aspect updatedAspect = DeleteEntityUtils.getAspectWithReferenceRemoved("hello", aspect, schema, new PathSpec("key_c", "key_a")); - assertTrue(updatedAspect.data().containsKey("key_c")); - assertEquals(aspect.data().get("key_c"), updatedAspect.data().get("key_c")); + assertFalse(updatedAspect.data().containsKey("key_c")); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java new file mode 100644 index 00000000000000..a1fe62c0a594de --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java @@ -0,0 +1,43 @@ +package com.linkedin.metadata.entity; + +import com.linkedin.metadata.EbeanTestUtils; +import com.linkedin.metadata.entity.ebean.EbeanAspectDao; +import com.linkedin.metadata.entity.ebean.EbeanRetentionService; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.models.registry.EntityRegistryException; +import io.ebean.EbeanServer; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static org.mockito.Mockito.*; + + +public class EbeanAspectMigrationsDaoTest extends AspectMigrationsDaoTest { + + public EbeanAspectMigrationsDaoTest() throws EntityRegistryException { + } + + @BeforeMethod + public void setupTest() { + EbeanServer server = EbeanTestUtils.createTestServer(); + _mockProducer = mock(EventProducer.class); + EbeanAspectDao dao = new EbeanAspectDao(server); + dao.setConnectionValidated(true); + _entityService = new EntityService(dao, _mockProducer, _testEntityRegistry); + _retentionService = new EbeanRetentionService(_entityService, server, 1000); + _entityService.setRetentionService(_retentionService); + + _migrationsDao = dao; + } + + /** + * Ideally, all tests would be in the base class, so they're reused between all implementations. + * When that's the case - test runner will ignore this class (and its base!) so we keep this dummy test + * to make sure this class will always be discovered. + */ + @Test + public void obligatoryTest() throws AssertionError { + Assert.assertTrue(true); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index cf6a9b997d0868..8fece4ae785119 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -1,9 +1,11 @@ package com.linkedin.metadata.entity; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; @@ -55,39 +57,37 @@ public void setupTest() { * to make sure this class will always be discovered. */ @Test - public void obligatoryTest() throws Exception { + public void obligatoryTest() throws AssertionError { Assert.assertTrue(true); } @Override @Test - public void testIngestListLatestAspects() throws Exception { + public void testIngestListLatestAspects() throws AssertionError { // TODO: If you're modifying this test - match your changes in sibling implementations. // TODO: Move this test into the base class, // If you can find a way for Cassandra and relational databases to share result ordering rules. - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:test2"); - Urn entityUrn3 = Urn.createFromString("urn:li:corpuser:test3"); + Urn entityUrn1 = UrnUtils.getUrn("urn:li:corpuser:test1"); + Urn entityUrn2 = UrnUtils.getUrn("urn:li:corpuser:test2"); + Urn entityUrn3 = UrnUtils.getUrn("urn:li:corpuser:test3"); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email2@test.com"); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 - CorpUserInfo writeAspect3 = createCorpUserInfo("email3@test.com"); + CorpUserInfo writeAspect3 = AspectGenerationUtils.createCorpUserInfo("email3@test.com"); _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // List aspects @@ -108,57 +108,55 @@ public void testIngestListLatestAspects() throws Exception { @Override @Test - public void testIngestListUrns() throws Exception { + public void testIngestListUrns() throws AssertionError { // TODO: If you're modifying this test - match your changes in sibling implementations. // TODO: Move this test into the base class, // If you can find a way for Cassandra and relational databases to share result ordering rules. - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:test2"); - Urn entityUrn3 = Urn.createFromString("urn:li:corpuser:test3"); + Urn entityUrn1 = UrnUtils.getUrn("urn:li:corpuser:test1"); + Urn entityUrn2 = UrnUtils.getUrn("urn:li:corpuser:test2"); + Urn entityUrn3 = UrnUtils.getUrn("urn:li:corpuser:test3"); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserKey().schema()); // Ingest CorpUserInfo Aspect #1 - RecordTemplate writeAspect1 = createCorpUserKey(entityUrn1); + RecordTemplate writeAspect1 = AspectGenerationUtils.createCorpUserKey(entityUrn1); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 - RecordTemplate writeAspect2 = createCorpUserKey(entityUrn2); + RecordTemplate writeAspect2 = AspectGenerationUtils.createCorpUserKey(entityUrn2); _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 - RecordTemplate writeAspect3 = createCorpUserKey(entityUrn3); + RecordTemplate writeAspect3 = AspectGenerationUtils.createCorpUserKey(entityUrn3); _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // List aspects urns ListUrnsResult batch1 = _entityService.listUrns(entityUrn1.getEntityType(), 0, 2); - assertEquals((int) batch1.getStart(), 0); - assertEquals((int) batch1.getCount(), 2); - assertEquals((int) batch1.getTotal(), 3); + assertEquals(batch1.getStart().intValue(), 0); + assertEquals(batch1.getCount().intValue(), 2); + assertEquals(batch1.getTotal().intValue(), 3); assertEquals(batch1.getEntities().size(), 2); assertEquals(entityUrn1.toString(), batch1.getEntities().get(0).toString()); assertEquals(entityUrn2.toString(), batch1.getEntities().get(1).toString()); ListUrnsResult batch2 = _entityService.listUrns(entityUrn1.getEntityType(), 2, 2); - assertEquals((int) batch2.getStart(), 2); - assertEquals((int) batch2.getCount(), 1); - assertEquals((int) batch2.getTotal(), 3); + assertEquals(batch2.getStart().intValue(), 2); + assertEquals(batch2.getCount().intValue(), 1); + assertEquals(batch2.getTotal().intValue(), 3); assertEquals(batch2.getEntities().size(), 1); assertEquals(entityUrn3.toString(), batch2.getEntities().get(0).toString()); } @Override @Test - public void testNestedTransactions() throws Exception { + public void testNestedTransactions() throws AssertionError { EbeanServer server = _aspectDao.getServer(); try (Transaction transaction = server.beginTransaction(TxScope.requiresNew() diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index d324165ad604e1..e5edcf22439d8c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -10,6 +10,7 @@ import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.ByteString; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.JacksonDataTemplateCodec; @@ -20,6 +21,7 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.events.metadata.ChangeType; import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.aspect.Aspect; import com.linkedin.metadata.aspect.CorpUserAspect; import com.linkedin.metadata.aspect.CorpUserAspectArray; @@ -34,8 +36,6 @@ import com.linkedin.metadata.run.AspectRowSummary; import com.linkedin.metadata.snapshot.CorpUserSnapshot; import com.linkedin.metadata.snapshot.Snapshot; -import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.MetadataAuditOperation; import com.linkedin.mxe.MetadataChangeLog; @@ -45,11 +45,6 @@ import com.linkedin.retention.Retention; import com.linkedin.retention.VersionBasedRetention; import com.linkedin.util.Pair; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import org.testng.annotations.Test; - -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -57,16 +52,14 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import javax.annotation.Nonnull; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.testng.annotations.Test; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; /** * A class to test {@link EntityService} @@ -88,7 +81,7 @@ abstract public class EntityServiceTest> pairToIngest = new ArrayList<>(); Status writeAspect1 = new Status().setRemoved(false); - String aspectName1 = getAspectName(writeAspect1); + String aspectName1 = AspectGenerationUtils.getAspectName(writeAspect1); pairToIngest.add(getAspectRecordPair(writeAspect1, Status.class)); - CorpUserInfo writeAspect2 = createCorpUserInfo("email@test.com"); - String aspectName2 = getAspectName(writeAspect2); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); + String aspectName2 = AspectGenerationUtils.getAspectName(writeAspect2); pairToIngest.add(getAspectRecordPair(writeAspect2, CorpUserInfo.class)); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); _entityService.ingestAspects(entityUrn, pairToIngest, TEST_AUDIT_STAMP, metadata1); @@ -450,7 +422,7 @@ public void testIngestAspectsGetLatestAspects() throws Exception { @Test public void testIngestTimeseriesAspect() throws Exception { - Urn entityUrn = Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)"); + Urn entityUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)"); DatasetProfile datasetProfile = new DatasetProfile(); datasetProfile.setRowCount(1000); datasetProfile.setColumnCount(15); @@ -470,15 +442,15 @@ public void testIngestTimeseriesAspect() throws Exception { } @Test - public void testUpdateGetAspect() throws Exception { + public void testUpdateGetAspect() throws AssertionError { // Test Writing a CorpUser Entity - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test"); + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test"); - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); AspectSpec corpUserInfoSpec = _testEntityRegistry.getEntitySpec("corpuser").getAspectSpec("corpUserInfo"); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect = AspectGenerationUtils.createCorpUserInfo("email@test.com"); // Validate retrieval of CorpUserInfo Aspect #1 _entityService.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, @@ -500,15 +472,15 @@ public void testUpdateGetAspect() throws Exception { } @Test - public void testGetAspectAtVersion() throws Exception { + public void testGetAspectAtVersion() throws AssertionError { // Test Writing a CorpUser Entity - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test"); + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test"); - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); AspectSpec corpUserInfoSpec = _testEntityRegistry.getEntitySpec("corpuser").getAspectSpec("corpUserInfo"); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect = AspectGenerationUtils.createCorpUserInfo("email@test.com"); // Validate retrieval of CorpUserInfo Aspect #1 _entityService.updateAspect(entityUrn, "corpuser", aspectName, corpUserInfoSpec, writeAspect, TEST_AUDIT_STAMP, 1, @@ -533,35 +505,30 @@ public void testGetAspectAtVersion() throws Exception { } @Test - public void testRollbackAspect() throws Exception { - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:test2"); - Urn entityUrn3 = Urn.createFromString("urn:li:corpuser:test3"); + public void testRollbackAspect() throws AssertionError { + Urn entityUrn1 = UrnUtils.getUrn("urn:li:corpuser:test1"); + Urn entityUrn2 = UrnUtils.getUrn("urn:li:corpuser:test2"); + Urn entityUrn3 = UrnUtils.getUrn("urn:li:corpuser:test3"); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); + SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); - SystemMetadata metadata2 = new SystemMetadata(); - metadata2.setLastObserved(1635792689); - metadata2.setRunId("run-456"); - - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email2@test.com"); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 - CorpUserInfo writeAspect3 = createCorpUserInfo("email3@test.com"); + CorpUserInfo writeAspect3 = AspectGenerationUtils.createCorpUserInfo("email3@test.com"); _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #1 Overwrite - CorpUserInfo writeAspect1Overwrite = createCorpUserInfo("email1.overwrite@test.com"); + CorpUserInfo writeAspect1Overwrite = AspectGenerationUtils.createCorpUserInfo("email1.overwrite@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); // this should no-op since this run has been overwritten @@ -593,29 +560,24 @@ public void testRollbackAspect() throws Exception { } @Test - public void testRollbackKey() throws Exception { - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + public void testRollbackKey() throws AssertionError { + Urn entityUrn1 = UrnUtils.getUrn("urn:li:corpuser:test1"); - SystemMetadata metadata2 = new SystemMetadata(); - metadata2.setLastObserved(1635792689); - metadata2.setRunId("run-456"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); + SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); String keyAspectName = _entityService.getKeyAspectName(entityUrn1); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); RecordTemplate writeKey1 = _entityService.buildKeyAspect(entityUrn1); _entityService.ingestAspect(entityUrn1, keyAspectName, writeKey1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #1 Overwrite - CorpUserInfo writeAspect1Overwrite = createCorpUserInfo("email1.overwrite@test.com"); + CorpUserInfo writeAspect1Overwrite = AspectGenerationUtils.createCorpUserInfo("email1.overwrite@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); // this should no-op since the key should have been written in the furst run @@ -647,39 +609,34 @@ public void testRollbackKey() throws Exception { } @Test - public void testRollbackUrn() throws Exception { - Urn entityUrn1 = Urn.createFromString("urn:li:corpuser:test1"); - Urn entityUrn2 = Urn.createFromString("urn:li:corpuser:test2"); - Urn entityUrn3 = Urn.createFromString("urn:li:corpuser:test3"); - - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + public void testRollbackUrn() throws AssertionError { + Urn entityUrn1 = UrnUtils.getUrn("urn:li:corpuser:test1"); + Urn entityUrn2 = UrnUtils.getUrn("urn:li:corpuser:test2"); + Urn entityUrn3 = UrnUtils.getUrn("urn:li:corpuser:test3"); - SystemMetadata metadata2 = new SystemMetadata(); - metadata2.setLastObserved(1635792689); - metadata2.setRunId("run-456"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); + SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); String keyAspectName = _entityService.getKeyAspectName(entityUrn1); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); RecordTemplate writeKey1 = _entityService.buildKeyAspect(entityUrn1); _entityService.ingestAspect(entityUrn1, keyAspectName, writeKey1, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email2@test.com"); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); _entityService.ingestAspect(entityUrn2, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #3 - CorpUserInfo writeAspect3 = createCorpUserInfo("email3@test.com"); + CorpUserInfo writeAspect3 = AspectGenerationUtils.createCorpUserInfo("email3@test.com"); _entityService.ingestAspect(entityUrn3, aspectName, writeAspect3, TEST_AUDIT_STAMP, metadata1); // Ingest CorpUserInfo Aspect #1 Overwrite - CorpUserInfo writeAspect1Overwrite = createCorpUserInfo("email1.overwrite@test.com"); + CorpUserInfo writeAspect1Overwrite = AspectGenerationUtils.createCorpUserInfo("email1.overwrite@test.com"); _entityService.ingestAspect(entityUrn1, aspectName, writeAspect1Overwrite, TEST_AUDIT_STAMP, metadata2); // this should no-op since the key should have been written in the furst run @@ -689,7 +646,7 @@ public void testRollbackUrn() throws Exception { rollbackKeyWithWrongRunId.setUrn(entityUrn1.toString()); // this should delete all related aspects - _entityService.deleteUrn(Urn.createFromString("urn:li:corpuser:test1")); + _entityService.deleteUrn(UrnUtils.getUrn("urn:li:corpuser:test1")); // assert the new most recent aspect is null RecordTemplate readNewRecentAspect = _entityService.getAspect(entityUrn1, aspectName, 0); @@ -700,20 +657,15 @@ public void testRollbackUrn() throws Exception { } @Test - public void testIngestGetLatestAspect() throws Exception { - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test"); + public void testIngestGetLatestAspect() throws AssertionError { + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test"); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); - String aspectName = PegasusUtils.getAspectNameFromSchema(writeAspect1.schema()); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); + String aspectName = AspectGenerationUtils.getAspectName(writeAspect1); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); - - SystemMetadata metadata2 = new SystemMetadata(); - metadata2.setLastObserved(1635792689); - metadata2.setRunId("run-456"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); + SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); // Validate retrieval of CorpUserInfo Aspect #1 _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); @@ -736,7 +688,7 @@ public void testIngestGetLatestAspect() throws Exception { reset(_mockProducer); // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email2@test.com"); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 _entityService.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); @@ -763,19 +715,14 @@ public void testIngestGetLatestAspect() throws Exception { @Test public void testIngestGetLatestEnvelopedAspect() throws Exception { - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test"); + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test"); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); - String aspectName = PegasusUtils.getAspectNameFromSchema(writeAspect1.schema()); - - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); + String aspectName = AspectGenerationUtils.getAspectName(writeAspect1); - SystemMetadata metadata2 = new SystemMetadata(); - metadata2.setLastObserved(1635792689); - metadata2.setRunId("run-456"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); + SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); // Validate retrieval of CorpUserInfo Aspect #1 _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); @@ -783,7 +730,7 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { assertTrue(DataTemplateUtil.areEqual(writeAspect1, new CorpUserInfo(readAspect1.getValue().data()))); // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email2@test.com"); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email2@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 _entityService.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); @@ -808,20 +755,16 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { } @Test - public void testIngestSameAspect() throws Exception { - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test"); + public void testIngestSameAspect() throws AssertionError { + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test"); // Ingest CorpUserInfo Aspect #1 - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); - String aspectName = PegasusUtils.getAspectNameFromSchema(writeAspect1.schema()); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); + String aspectName = AspectGenerationUtils.getAspectName(writeAspect1); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); - - SystemMetadata metadata2 = new SystemMetadata(); - metadata2.setLastObserved(1635792689); - metadata2.setRunId("run-456"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); + SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); + SystemMetadata metadata3 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-123"); // Validate retrieval of CorpUserInfo Aspect #1 _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); @@ -844,21 +787,17 @@ public void testIngestSameAspect() throws Exception { reset(_mockProducer); // Ingest CorpUserInfo Aspect #2 - CorpUserInfo writeAspect2 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect2 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); // Validate retrieval of CorpUserInfo Aspect #2 _entityService.ingestAspect(entityUrn, aspectName, writeAspect2, TEST_AUDIT_STAMP, metadata2); RecordTemplate readAspect2 = _entityService.getLatestAspect(entityUrn, aspectName); - EntityAspect readAspectDao2 = _aspectDao.getAspect(entityUrn.toString(), aspectName, 0); + EntityAspect readAspectDao2 = _aspectDao.getAspect(entityUrn.toString(), aspectName, ASPECT_LATEST_VERSION); assertTrue(DataTemplateUtil.areEqual(writeAspect2, readAspect2)); assertFalse(DataTemplateUtil.areEqual(EntityUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata2)); assertFalse(DataTemplateUtil.areEqual(EntityUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata1)); - SystemMetadata metadata3 = new SystemMetadata(); - metadata3.setLastObserved(1635792689); - metadata3.setRunId("run-123"); - assertTrue(DataTemplateUtil.areEqual(EntityUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata3)); verify(_mockProducer, times(0)).produceMetadataChangeLog(Mockito.eq(entityUrn), Mockito.any(), mclCaptor.capture()); @@ -870,24 +809,22 @@ public void testIngestSameAspect() throws Exception { } @Test - public void testRetention() throws Exception { - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test1"); + public void testRetention() throws AssertionError { + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test1"); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); // Ingest CorpUserInfo Aspect - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityService.ingestAspect(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - CorpUserInfo writeAspect1a = createCorpUserInfo("email_a@test.com"); + CorpUserInfo writeAspect1a = AspectGenerationUtils.createCorpUserInfo("email_a@test.com"); _entityService.ingestAspect(entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); - CorpUserInfo writeAspect1b = createCorpUserInfo("email_b@test.com"); + CorpUserInfo writeAspect1b = AspectGenerationUtils.createCorpUserInfo("email_b@test.com"); _entityService.ingestAspect(entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); - String aspectName2 = PegasusUtils.getAspectNameFromSchema(new Status().schema()); + String aspectName2 = AspectGenerationUtils.getAspectName(new Status()); // Ingest Status Aspect Status writeAspect2 = new Status().setRemoved(true); _entityService.ingestAspect(entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); @@ -905,7 +842,7 @@ public void testRetention() throws Exception { new Retention().setVersion(new VersionBasedRetention().setMaxVersions(4)))); // Ingest CorpUserInfo Aspect again - CorpUserInfo writeAspect1c = createCorpUserInfo("email_c@test.com"); + CorpUserInfo writeAspect1c = AspectGenerationUtils.createCorpUserInfo("email_c@test.com"); _entityService.ingestAspect(entityUrn, aspectName, writeAspect1c, TEST_AUDIT_STAMP, metadata1); // Ingest Status Aspect again Status writeAspect2c = new Status().setRemoved(false); @@ -925,24 +862,22 @@ public void testRetention() throws Exception { } @Test - public void testIngestAspectIfNotPresent() throws Exception { - Urn entityUrn = Urn.createFromString("urn:li:corpuser:test1"); + public void testIngestAspectIfNotPresent() throws AssertionError { + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:test1"); - SystemMetadata metadata1 = new SystemMetadata(); - metadata1.setLastObserved(1625792689); - metadata1.setRunId("run-123"); + SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(); - String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + String aspectName = AspectGenerationUtils.getAspectName(new CorpUserInfo()); // Ingest CorpUserInfo Aspect - CorpUserInfo writeAspect1 = createCorpUserInfo("email@test.com"); + CorpUserInfo writeAspect1 = AspectGenerationUtils.createCorpUserInfo("email@test.com"); _entityService.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1, TEST_AUDIT_STAMP, metadata1); - CorpUserInfo writeAspect1a = createCorpUserInfo("email_a@test.com"); + CorpUserInfo writeAspect1a = AspectGenerationUtils.createCorpUserInfo("email_a@test.com"); _entityService.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1a, TEST_AUDIT_STAMP, metadata1); - CorpUserInfo writeAspect1b = createCorpUserInfo("email_b@test.com"); + CorpUserInfo writeAspect1b = AspectGenerationUtils.createCorpUserInfo("email_b@test.com"); _entityService.ingestAspectIfNotPresent(entityUrn, aspectName, writeAspect1b, TEST_AUDIT_STAMP, metadata1); - String aspectName2 = PegasusUtils.getAspectNameFromSchema(new Status().schema()); + String aspectName2 = AspectGenerationUtils.getAspectName(new Status()); // Ingest Status Aspect Status writeAspect2 = new Status().setRemoved(true); _entityService.ingestAspectIfNotPresent(entityUrn, aspectName2, writeAspect2, TEST_AUDIT_STAMP, metadata1); @@ -961,14 +896,6 @@ public void testIngestAspectIfNotPresent() throws Exception { assertEquals(_entityService.listLatestAspects(entityUrn.getEntityType(), aspectName2, 0, 10).getTotalCount(), 1); } - protected static AuditStamp createTestAuditStamp() { - try { - return new AuditStamp().setTime(123L).setActor(Urn.createFromString("urn:li:principal:tester")); - } catch (Exception e) { - throw new RuntimeException("Failed to create urn"); - } - } - @Nonnull protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String email) throws Exception { CorpuserUrn corpuserUrn = CorpuserUrn.createFromUrn(entityUrn); @@ -976,7 +903,7 @@ protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String Snapshot snapshot = new Snapshot(); CorpUserSnapshot corpUserSnapshot = new CorpUserSnapshot(); List userAspects = new ArrayList<>(); - userAspects.add(CorpUserAspect.create(createCorpUserInfo(email))); + userAspects.add(CorpUserAspect.create(AspectGenerationUtils.createCorpUserInfo(email))); corpUserSnapshot.setAspects(new CorpUserAspectArray(userAspects)); corpUserSnapshot.setUrn(corpuserUrn); snapshot.setCorpUserSnapshot(corpUserSnapshot); @@ -984,28 +911,11 @@ protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String return entity; } - @Nonnull - protected RecordTemplate createCorpUserKey(Urn urn) throws Exception { - return EntityKeyUtils.convertUrnToEntityKey(urn, new CorpUserKey().schema()); - } - - @Nonnull - protected CorpUserInfo createCorpUserInfo(String email) throws Exception { - CorpUserInfo corpUserInfo = new CorpUserInfo(); - corpUserInfo.setEmail(email); - corpUserInfo.setActive(true); - return corpUserInfo; - } - - protected String getAspectName(RecordTemplate record) { - return PegasusUtils.getAspectNameFromSchema(record.schema()); - } - protected Pair getAspectRecordPair(T aspect, Class clazz) throws Exception { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); RecordTemplate recordTemplate = RecordUtils.toRecordTemplate(clazz, objectMapper.writeValueAsString(aspect)); - return new Pair<>(getAspectName(aspect), recordTemplate); + return new Pair<>(AspectGenerationUtils.getAspectName(aspect), recordTemplate); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java new file mode 100644 index 00000000000000..d566e8b0bec338 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java @@ -0,0 +1,475 @@ +package com.linkedin.metadata.graph.sibling; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.Siblings; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.EntityLineageResult; +import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.graph.LineageDirection; +import com.linkedin.metadata.graph.LineageRelationship; +import com.linkedin.metadata.graph.LineageRelationshipArray; +import com.linkedin.metadata.graph.SiblingGraphService; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import org.mockito.Mockito; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.*; + + +public class SiblingGraphServiceTest { + + /** + * Some test URN types. + */ + protected static String datasetType = "dataset"; + protected static String userType = "user"; + + /** + * Some test datasets. + */ + protected static String datasetOneUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetOne,PROD)"; + protected static String datasetTwoUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetTwo,PROD)"; + protected static String datasetThreeUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetThree,PROD)"; + protected static String datasetFourUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetFour,PROD)"; + protected static String datasetFiveUrnString = "urn:li:" + datasetType + ":(urn:li:dataPlatform:type,SampleDatasetFive,PROD)"; + + protected static Urn datasetOneUrn = createFromString(datasetOneUrnString); + protected static Urn datasetTwoUrn = createFromString(datasetTwoUrnString); + protected static Urn datasetThreeUrn = createFromString(datasetThreeUrnString); + protected static Urn datasetFourUrn = createFromString(datasetFourUrnString); + protected static Urn datasetFiveUrn = createFromString(datasetFiveUrnString); + + + /** + * Some test relationships. + */ + protected static String downstreamOf = "DownstreamOf"; + + private GraphService _graphService; + private SiblingGraphService _client; + EntityService _mockEntityService; + + @BeforeClass + public void setup() { + _mockEntityService = Mockito.mock(EntityService.class); + _graphService = Mockito.mock(GraphService.class); + _client = new SiblingGraphService(_mockEntityService, _graphService); + } + + @Test + public void testNoSiblingMetadata() throws Exception { + EntityLineageResult mockResult = new EntityLineageResult(); + LineageRelationshipArray relationships = new LineageRelationshipArray(); + LineageRelationship relationship1 = new LineageRelationship(); + relationship1.setDegree(0); + relationship1.setType(downstreamOf); + relationship1.setEntity(datasetOneUrn); + + LineageRelationship relationship2 = new LineageRelationship(); + relationship2.setDegree(0); + relationship2.setType(downstreamOf); + relationship2.setEntity(datasetTwoUrn); + + LineageRelationship relationship3 = new LineageRelationship(); + relationship3.setDegree(0); + relationship3.setType(downstreamOf); + relationship3.setEntity(datasetThreeUrn); + + relationships.add(relationship1); + relationships.add(relationship2); + relationships.add(relationship3); + + mockResult.setStart(0); + mockResult.setTotal(200); + mockResult.setCount(3); + mockResult.setRelationships(relationships); + + Mockito.when(_graphService.getLineage( + datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1 + )).thenReturn(mockResult); + + Mockito.when(_mockEntityService.getLatestAspect(datasetFourUrn, SIBLINGS_ASPECT_NAME)).thenReturn(null); + + SiblingGraphService service = _client; + + EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1); + + // assert sibling graph service is a pass through in the case that there is no sibling metadataa + assertEquals(upstreamLineage, mockResult); + } + + @Test + public void testNoSiblingInResults() throws Exception { + EntityLineageResult mockResult = new EntityLineageResult(); + EntityLineageResult siblingMockResult = new EntityLineageResult(); + + LineageRelationshipArray relationships = new LineageRelationshipArray(); + LineageRelationship relationship1 = new LineageRelationship(); + relationship1.setDegree(0); + relationship1.setType(downstreamOf); + relationship1.setEntity(datasetOneUrn); + + LineageRelationship relationship2 = new LineageRelationship(); + relationship2.setDegree(0); + relationship2.setType(downstreamOf); + relationship2.setEntity(datasetTwoUrn); + + LineageRelationship relationship3 = new LineageRelationship(); + relationship3.setDegree(0); + relationship3.setType(downstreamOf); + relationship3.setEntity(datasetThreeUrn); + + relationships.add(relationship1); + relationships.add(relationship2); + relationships.add(relationship3); + + mockResult.setStart(0); + mockResult.setTotal(200); + mockResult.setCount(3); + mockResult.setRelationships(relationships); + + Mockito.when(_graphService.getLineage( + datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1 + )).thenReturn(mockResult); + + siblingMockResult.setStart(0); + siblingMockResult.setTotal(0); + siblingMockResult.setCount(0); + siblingMockResult.setRelationships(new LineageRelationshipArray()); + + Mockito.when(_graphService.getLineage( + datasetFiveUrn, LineageDirection.UPSTREAM, 0, 97, 1 + )).thenReturn(siblingMockResult); + + Siblings noRelevantSiblingsResponse = new Siblings(); + noRelevantSiblingsResponse.setPrimary(true); + noRelevantSiblingsResponse.setSiblings(new UrnArray(ImmutableList.of(datasetFiveUrn))); + + Mockito.when(_mockEntityService.getLatestAspect(datasetFourUrn, SIBLINGS_ASPECT_NAME)).thenReturn(noRelevantSiblingsResponse); + + Siblings dataset1Siblings = new Siblings(); + dataset1Siblings.setPrimary(false); + dataset1Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset2Siblings = new Siblings(); + dataset2Siblings.setPrimary(false); + dataset2Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset3Siblings = new Siblings(); + dataset3Siblings.setPrimary(false); + dataset3Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Map> siblingsMap = ImmutableMap.of( + datasetOneUrn, ImmutableList.of(dataset1Siblings), + datasetTwoUrn, ImmutableList.of(dataset2Siblings), + datasetThreeUrn, ImmutableList.of(dataset3Siblings) + ); + + Mockito.when(_mockEntityService.getLatestAspects(Mockito.any(), Mockito.any())).thenReturn(siblingsMap); + + SiblingGraphService service = _client; + + EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1); + + // assert sibling graph service is a pass through in the case that your sibling has no lineage + assertEquals(upstreamLineage, mockResult); + } + + @Test + public void testSiblingInResult() throws Exception { + EntityLineageResult mockResult = new EntityLineageResult(); + EntityLineageResult siblingMockResult = new EntityLineageResult(); + + LineageRelationshipArray relationships = new LineageRelationshipArray(); + LineageRelationship relationship1 = new LineageRelationship(); + relationship1.setDegree(0); + relationship1.setType(downstreamOf); + relationship1.setEntity(datasetOneUrn); + + LineageRelationship relationship2 = new LineageRelationship(); + relationship2.setDegree(0); + relationship2.setType(downstreamOf); + relationship2.setEntity(datasetTwoUrn); + + LineageRelationship relationship3 = new LineageRelationship(); + relationship3.setDegree(0); + relationship3.setType(downstreamOf); + relationship3.setEntity(datasetThreeUrn); + + relationships.add(relationship1); + relationships.add(relationship2); + relationships.add(relationship3); + + mockResult.setStart(0); + mockResult.setTotal(3); + mockResult.setCount(3); + mockResult.setRelationships(relationships); + + siblingMockResult.setStart(0); + siblingMockResult.setTotal(0); + siblingMockResult.setCount(0); + siblingMockResult.setRelationships(new LineageRelationshipArray()); + + Mockito.when(_graphService.getLineage( + datasetThreeUrn, LineageDirection.UPSTREAM, 0, 98, 1 + )).thenReturn(siblingMockResult); + + + Mockito.when(_graphService.getLineage( + datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1 + )).thenReturn(mockResult); + + Siblings siblingInSearchResult = new Siblings(); + siblingInSearchResult.setPrimary(true); + siblingInSearchResult.setSiblings(new UrnArray(ImmutableList.of(datasetThreeUrn))); + + Mockito.when(_mockEntityService.getLatestAspect(datasetFourUrn, SIBLINGS_ASPECT_NAME)).thenReturn(siblingInSearchResult); + + Siblings dataset1Siblings = new Siblings(); + dataset1Siblings.setPrimary(false); + dataset1Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset2Siblings = new Siblings(); + dataset2Siblings.setPrimary(false); + dataset2Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset3Siblings = new Siblings(); + dataset3Siblings.setPrimary(false); + dataset3Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Map> siblingsMap = ImmutableMap.of( + datasetOneUrn, ImmutableList.of(dataset1Siblings), + datasetTwoUrn, ImmutableList.of(dataset2Siblings), + datasetThreeUrn, ImmutableList.of(dataset3Siblings) + ); + + Mockito.when(_mockEntityService.getLatestAspects(Mockito.any(), Mockito.any())).thenReturn(siblingsMap); + + SiblingGraphService service = _client; + + EntityLineageResult expectedResult = mockResult.clone(); + expectedResult.setTotal(3); + expectedResult.setCount(2); + expectedResult.setRelationships(new LineageRelationshipArray(relationship1, relationship2)); + + EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1); + + // assert your sibling will be filtered out of your lineage + assertEquals(upstreamLineage, expectedResult); + assertEquals(upstreamLineage.getRelationships().size(), 2); + } + + @Test + public void testCombineSiblingResult() throws Exception { + EntityLineageResult mockResult = new EntityLineageResult(); + EntityLineageResult siblingMockResult = new EntityLineageResult(); + EntityLineageResult expectedResult = new EntityLineageResult(); + + LineageRelationshipArray relationships = new LineageRelationshipArray(); + LineageRelationshipArray siblingRelationships = new LineageRelationshipArray(); + LineageRelationshipArray expectedRelationships = new LineageRelationshipArray(); + + LineageRelationship relationship1 = new LineageRelationship(); + relationship1.setDegree(0); + relationship1.setType(downstreamOf); + relationship1.setEntity(datasetOneUrn); + + LineageRelationship relationship2 = new LineageRelationship(); + relationship2.setDegree(0); + relationship2.setType(downstreamOf); + relationship2.setEntity(datasetTwoUrn); + + LineageRelationship relationship3 = new LineageRelationship(); + relationship3.setDegree(0); + relationship3.setType(downstreamOf); + relationship3.setEntity(datasetThreeUrn); + + LineageRelationship relationship4 = new LineageRelationship(); + relationship4.setDegree(0); + relationship4.setType(downstreamOf); + relationship4.setEntity(datasetFiveUrn); + + relationships.add(relationship1); + + expectedRelationships.add(relationship2); + expectedRelationships.add(relationship4); + expectedRelationships.add(relationship1); + + expectedResult.setCount(3); + expectedResult.setStart(0); + expectedResult.setTotal(3); + expectedResult.setRelationships(expectedRelationships); + + mockResult.setStart(0); + mockResult.setTotal(1); + mockResult.setCount(1); + mockResult.setRelationships(relationships); + + siblingRelationships.add(relationship2); + siblingRelationships.add(relationship4); + siblingMockResult.setStart(0); + siblingMockResult.setTotal(2); + siblingMockResult.setCount(2); + siblingMockResult.setRelationships(siblingRelationships); + + Mockito.when(_graphService.getLineage( + datasetThreeUrn, LineageDirection.UPSTREAM, 0, 99, 1 + )).thenReturn(siblingMockResult); + + + Mockito.when(_graphService.getLineage( + datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1 + )).thenReturn(mockResult); + + Siblings siblingInSearchResult = new Siblings(); + siblingInSearchResult.setPrimary(true); + siblingInSearchResult.setSiblings(new UrnArray(ImmutableList.of(datasetThreeUrn))); + + Mockito.when(_mockEntityService.getLatestAspect(datasetFourUrn, SIBLINGS_ASPECT_NAME)).thenReturn(siblingInSearchResult); + + Siblings dataset1Siblings = new Siblings(); + dataset1Siblings.setPrimary(false); + dataset1Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset2Siblings = new Siblings(); + dataset2Siblings.setPrimary(false); + dataset2Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset3Siblings = new Siblings(); + dataset3Siblings.setPrimary(false); + dataset3Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Map> siblingsMap = ImmutableMap.of( + datasetOneUrn, ImmutableList.of(dataset1Siblings), + datasetTwoUrn, ImmutableList.of(dataset2Siblings), + datasetThreeUrn, ImmutableList.of(dataset3Siblings), + datasetFiveUrn, ImmutableList.of(dataset3Siblings) + ); + + Mockito.when(_mockEntityService.getLatestAspects(Mockito.any(), Mockito.any())).thenReturn(siblingsMap); + + SiblingGraphService service = _client; + + EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1); + + // assert your lineage will be combined with your siblings lineage + assertEquals(upstreamLineage, expectedResult); + } + + @Test + public void testUpstreamOfSiblings() throws Exception { + EntityLineageResult mockResult = new EntityLineageResult(); + EntityLineageResult siblingMockResult = new EntityLineageResult(); + EntityLineageResult expectedResult = new EntityLineageResult(); + + LineageRelationshipArray relationships = new LineageRelationshipArray(); + LineageRelationshipArray siblingRelationships = new LineageRelationshipArray(); + LineageRelationshipArray expectedRelationships = new LineageRelationshipArray(); + + LineageRelationship relationship1 = new LineageRelationship(); + relationship1.setDegree(0); + relationship1.setType(downstreamOf); + relationship1.setEntity(datasetOneUrn); + + LineageRelationship relationship2 = new LineageRelationship(); + relationship2.setDegree(0); + relationship2.setType(downstreamOf); + relationship2.setEntity(datasetTwoUrn); + + LineageRelationship relationship3 = new LineageRelationship(); + relationship3.setDegree(0); + relationship3.setType(downstreamOf); + relationship3.setEntity(datasetThreeUrn); + + LineageRelationship relationship4 = new LineageRelationship(); + relationship4.setDegree(0); + relationship4.setType(downstreamOf); + relationship4.setEntity(datasetFiveUrn); + + relationships.add(relationship1); + + expectedRelationships.add(relationship4); + expectedRelationships.add(relationship1); + + expectedResult.setCount(2); + expectedResult.setStart(0); + expectedResult.setTotal(3); + expectedResult.setRelationships(expectedRelationships); + + mockResult.setStart(0); + mockResult.setTotal(1); + mockResult.setCount(1); + mockResult.setRelationships(relationships); + + siblingRelationships.add(relationship2); + siblingRelationships.add(relationship4); + siblingMockResult.setStart(0); + siblingMockResult.setTotal(2); + siblingMockResult.setCount(2); + siblingMockResult.setRelationships(siblingRelationships); + + Mockito.when(_graphService.getLineage( + datasetThreeUrn, LineageDirection.UPSTREAM, 0, 99, 1 + )).thenReturn(siblingMockResult); + + + Mockito.when(_graphService.getLineage( + datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1 + )).thenReturn(mockResult); + + Siblings siblingInSearchResult = new Siblings(); + siblingInSearchResult.setPrimary(true); + siblingInSearchResult.setSiblings(new UrnArray(ImmutableList.of(datasetThreeUrn))); + + Mockito.when(_mockEntityService.getLatestAspect(datasetFourUrn, SIBLINGS_ASPECT_NAME)).thenReturn(siblingInSearchResult); + + Siblings dataset1Siblings = new Siblings(); + dataset1Siblings.setPrimary(false); + dataset1Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset2Siblings = new Siblings(); + dataset2Siblings.setPrimary(false); + dataset2Siblings.setSiblings(new UrnArray(ImmutableList.of(datasetFiveUrn))); + + Siblings dataset3Siblings = new Siblings(); + dataset3Siblings.setPrimary(false); + dataset3Siblings.setSiblings(new UrnArray(ImmutableList.of())); + + Siblings dataset5Siblings = new Siblings(); + dataset5Siblings.setPrimary(true); + dataset5Siblings.setSiblings(new UrnArray(ImmutableList.of(datasetTwoUrn))); + + Map> siblingsMap = ImmutableMap.of( + datasetOneUrn, ImmutableList.of(dataset1Siblings), + datasetTwoUrn, ImmutableList.of(dataset2Siblings), + datasetThreeUrn, ImmutableList.of(dataset3Siblings), + datasetFiveUrn, ImmutableList.of(dataset5Siblings) + ); + + Mockito.when(_mockEntityService.getLatestAspects(Mockito.any(), Mockito.any())).thenReturn(siblingsMap); + + SiblingGraphService service = _client; + + EntityLineageResult upstreamLineage = service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1); + + // assert your lineage will not contain two siblings + assertEquals(upstreamLineage, expectedResult); + } + + static Urn createFromString(@Nonnull String rawUrn) { + try { + return Urn.createFromString(rawUrn); + } catch (URISyntaxException e) { + return null; + } + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java index 399f55e5307143..524988346f80ef 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchServiceTest.java @@ -16,6 +16,10 @@ import com.linkedin.metadata.graph.LineageRelationshipArray; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; +import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; +import com.linkedin.metadata.search.cache.CachingAllEntitiesSearchAggregator; +import com.linkedin.metadata.search.cache.EntityDocCountCache; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchServiceTest; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; @@ -87,8 +91,17 @@ public void setup() { } private void resetService() { + CachingEntitySearchService cachingEntitySearchService = new CachingEntitySearchService(_cacheManager, _elasticSearchService, 100, true); _lineageSearchService = new LineageSearchService( - new SearchService(_entityRegistry, _elasticSearchService, new SimpleRanker(), _cacheManager, 100, true), + new SearchService( + new EntityDocCountCache(_entityRegistry, _elasticSearchService), + cachingEntitySearchService, + new CachingAllEntitiesSearchAggregator( + _cacheManager, + new AllEntitiesSearchAggregator(_entityRegistry, _elasticSearchService, cachingEntitySearchService, new SimpleRanker()), + 100, + true), + new SimpleRanker()), _graphService, _cacheManager.getCache("test")); } @@ -134,11 +147,11 @@ public void testSearchService() throws Exception { anyInt())).thenReturn(mockResult(Collections.emptyList())); LineageSearchResult searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), - "test", null, null, 0, 10); + "test", null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); clearCache(); @@ -147,11 +160,11 @@ public void testSearchService() throws Exception { mockResult(ImmutableList.of(new LineageRelationship().setEntity(TEST_URN).setType("test").setDegree(1)))); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), - "test", null, null, 0, 10); + "test", null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); clearCache(); @@ -168,7 +181,7 @@ public void testSearchService() throws Exception { anyInt())).thenReturn(mockResult(Collections.emptyList())); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); clearCache(); @@ -178,21 +191,21 @@ public void testSearchService() throws Exception { mockResult(ImmutableList.of(new LineageRelationship().setEntity(urn).setType("test").setDegree(1)))); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - QueryUtils.newFilter("degree.keyword", "1"), null, 0, 10); + null, QueryUtils.newFilter("degree.keyword", "1"), null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - QueryUtils.newFilter("degree.keyword", "2"), null, 0, 10); + null, QueryUtils.newFilter("degree.keyword", "2"), null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); clearCache(); @@ -208,7 +221,7 @@ public void testSearchService() throws Exception { searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); clearCache(); @@ -218,7 +231,7 @@ public void testSearchService() throws Exception { mockResult(ImmutableList.of(new LineageRelationship().setEntity(urn2).setType("test").setDegree(1)))); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); clearCache(); @@ -232,7 +245,7 @@ public void testSearchService() throws Exception { mockResult(ImmutableList.of(new LineageRelationship().setEntity(urn).setType("test").setDegree(1)))); searchResult = _lineageSearchService.searchAcrossLineage(TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), "test", - null, null, 0, 10); + null, null, null, 0, 10); assertEquals(searchResult.getNumEntities().intValue(), 0); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java index 7f587c5e6175b8..7a82864320a6e1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java @@ -9,6 +9,10 @@ import com.linkedin.metadata.ElasticTestUtils; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; +import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; +import com.linkedin.metadata.search.cache.CachingAllEntitiesSearchAggregator; +import com.linkedin.metadata.search.cache.EntityDocCountCache; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchServiceTest; import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; @@ -65,8 +69,24 @@ public void setup() { } private void resetSearchService() { - _searchService = - new SearchService(_entityRegistry, _elasticSearchService, new SimpleRanker(), _cacheManager, 100, true); + CachingEntitySearchService cachingEntitySearchService = new CachingEntitySearchService( + _cacheManager, + _elasticSearchService, + 100, + true); + _searchService = new SearchService( + new EntityDocCountCache(_entityRegistry, _elasticSearchService), + cachingEntitySearchService, + new CachingAllEntitiesSearchAggregator( + _cacheManager, + new AllEntitiesSearchAggregator( + _entityRegistry, + _elasticSearchService, + cachingEntitySearchService, + new SimpleRanker()), + 100, + true), + new SimpleRanker()); } @BeforeMethod diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java index 9ed1fc337e29fb..8038c5561a5d75 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java @@ -16,8 +16,9 @@ public void testMappingsBuilder() { Map result = MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec()); assertEquals(result.size(), 1); Map properties = (Map) result.get("properties"); - assertEquals(properties.size(), 14); + assertEquals(properties.size(), 15); assertEquals(properties.get("urn"), ImmutableMap.of("type", "keyword")); + assertEquals(properties.get("runId"), ImmutableMap.of("type", "keyword")); assertTrue(properties.containsKey("browsePaths")); // KEYWORD Map keyPart3Field = (Map) properties.get("keyPart3"); diff --git a/metadata-jobs/mae-consumer-job/README.md b/metadata-jobs/mae-consumer-job/README.md index 3fbc3ff5bc544c..3d75bd14379e4b 100644 --- a/metadata-jobs/mae-consumer-job/README.md +++ b/metadata-jobs/mae-consumer-job/README.md @@ -2,17 +2,27 @@ title: "metadata-jobs:mae-consumer-job" --- -# MetadataAuditEvent (MAE) Consumer Job -MAE Consumer is a [Kafka Streams](https://kafka.apache.org/documentation/streams/) job. Its main function is to listen -`MetadataAuditEvent` Kafka topic for messages and process those messages, converting changes in the metadata model into updates -against secondary search & graph indexes. +# Metadata Audit Event Consumer Job + +The Metadata Audit Event Consumer is a [Kafka Streams](https://kafka.apache.org/documentation/streams/) job which can be deployed by itself, or as part of the Metadata Service. + +Its main function is to listen to change log events emitted as a result of changes made to the Metadata Graph, converting changes in the metadata model into updates +against secondary search & graph indexes (among other things) + +Today the job consumes from two important Kafka topics: + +1. `MetadataChangeLog_Versioned_v1` +2. `MetadataChangeLog_Timeseries_v1` + +> Where does the name **Metadata Audit Event** come from? Well, history. Previously, this job consumed +> a single `MetadataAuditEvent` topic which has been deprecated and removed from the critical path. Hence, the name! ## Pre-requisites * You need to have [JDK8](https://www.oracle.com/java/technologies/jdk8-downloads.html) -installed on your machine to be able to build `DataHub GMS`. +installed on your machine to be able to build `DataHub Metadata Service`. ## Build -`MAE Consumer Job` is already built as part of top level build: +`Metadata Audit Event Consumer Job` is already built as part of top level build: ``` ./gradlew build ``` @@ -22,11 +32,11 @@ However, if you only want to build `MAE Consumer Job` specifically: ``` ## Dependencies -Before starting `MAE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup), +Before starting `Metadata Audit Event Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup), [Elasticsearch](../../docker/elasticsearch), and [Neo4j](../../docker/neo4j) Docker containers are up and running. ## Start via Docker image -Quickest way to try out `MAE Consumer Job` is running the [Docker image](../../docker/datahub-mae-consumer). +The quickest way to try out `Metadata Audit Event Consumer Job` is running the [Docker image](../../docker/datahub-mae-consumer). ## Start via command line If you do modify things and want to try it out quickly without building the Docker image, you can also run diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java index 2434717e8d13a6..02946464d5fab3 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java @@ -11,6 +11,7 @@ import com.linkedin.metadata.kafka.hook.UpdateIndicesHook; import com.linkedin.metadata.kafka.hook.event.EntityChangeEventGeneratorHook; import com.linkedin.metadata.kafka.hook.ingestion.IngestionSchedulerHook; +import com.linkedin.metadata.kafka.hook.siblings.SiblingAssociationHook; import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.Topics; @@ -30,7 +31,13 @@ @Slf4j @Component @Conditional(MetadataChangeLogProcessorCondition.class) -@Import({UpdateIndicesHook.class, IngestionSchedulerHook.class, EntityChangeEventGeneratorHook.class, KafkaEventConsumerFactory.class}) +@Import({ + UpdateIndicesHook.class, + IngestionSchedulerHook.class, + EntityChangeEventGeneratorHook.class, + KafkaEventConsumerFactory.class, + SiblingAssociationHook.class +}) @EnableKafka public class MetadataChangeLogProcessor { @@ -41,8 +48,10 @@ public class MetadataChangeLogProcessor { public MetadataChangeLogProcessor( @Nonnull final UpdateIndicesHook updateIndicesHook, @Nonnull final IngestionSchedulerHook ingestionSchedulerHook, - @Nonnull final EntityChangeEventGeneratorHook entityChangeEventHook) { - this.hooks = ImmutableList.of(updateIndicesHook, ingestionSchedulerHook, entityChangeEventHook); + @Nonnull final EntityChangeEventGeneratorHook entityChangeEventHook, + @Nonnull final SiblingAssociationHook siblingAssociationHook + ) { + this.hooks = ImmutableList.of(updateIndicesHook, ingestionSchedulerHook, entityChangeEventHook, siblingAssociationHook); this.hooks.forEach(MetadataChangeLogHook::init); } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java index 6e162af48fa57a..0ab79c2007ef49 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java @@ -44,6 +44,7 @@ import java.util.Optional; import java.util.Set; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Import; @@ -115,7 +116,7 @@ public void invoke(@Nonnull MetadataChangeLog event) { updateTimeseriesFields(event.getEntityType(), event.getAspectName(), urn, aspect, aspectSpec, event.getSystemMetadata()); } else { - updateSearchService(entitySpec.getName(), urn, aspectSpec, aspect); + updateSearchService(entitySpec.getName(), urn, aspectSpec, aspect, event.hasSystemMetadata() ? event.getSystemMetadata().getRunId() : null); updateGraphService(urn, aspectSpec, aspect); updateSystemMetadata(event.getSystemMetadata(), urn, aspectSpec, aspect); } @@ -185,7 +186,7 @@ private void updateGraphService(Urn urn, AspectSpec aspectSpec, RecordTemplate a /** * Process snapshot and update search index */ - private void updateSearchService(String entityName, Urn urn, AspectSpec aspectSpec, RecordTemplate aspect) { + private void updateSearchService(String entityName, Urn urn, AspectSpec aspectSpec, RecordTemplate aspect, @Nullable String runId) { Optional searchDocument; try { searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, false); @@ -277,7 +278,7 @@ private void deleteSearchData(Urn urn, String entityName, AspectSpec aspectSpec, Optional searchDocument; try { - searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, true); + searchDocument = _searchDocumentTransformer.transformAspect(urn, aspect, aspectSpec, true); // TODO } catch (Exception e) { log.error("Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); return; diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java new file mode 100644 index 00000000000000..d8808c5b5abeb8 --- /dev/null +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java @@ -0,0 +1,442 @@ +package com.linkedin.metadata.kafka.hook.siblings; + +import com.datahub.authentication.Authentication; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Siblings; +import com.linkedin.common.SubTypes; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.dataset.UpstreamArray; +import com.linkedin.dataset.UpstreamLineage; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; +import com.linkedin.gms.factory.entity.RestliEntityClientFactory; +import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; +import com.linkedin.gms.factory.search.EntitySearchServiceFactory; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.GenericAspect; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.inject.Singleton; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; + +import static com.linkedin.metadata.Constants.*; + + +/** + * This hook associates dbt datasets with their sibling entities + */ +@Slf4j +@Component +@Singleton +@Import({EntityRegistryFactory.class, RestliEntityClientFactory.class, EntitySearchServiceFactory.class, SystemAuthenticationFactory.class}) +public class SiblingAssociationHook implements MetadataChangeLogHook { + + public static final String SIBLING_ASSOCIATION_SYSTEM_ACTOR = "urn:li:corpuser:__datahub_system_sibling_hook"; + public static final String DBT_PLATFORM_NAME = "dbt"; + public static final String SOURCE_SUBTYPE = "source"; + + private final EntityRegistry _entityRegistry; + private final RestliEntityClient _entityClient; + private final EntitySearchService _searchService; + private final Authentication _systemAuthentication; + + @Autowired + public SiblingAssociationHook( + @Nonnull final EntityRegistry entityRegistry, + @Nonnull final RestliEntityClient entityClient, + @Nonnull final EntitySearchService searchService, + @Nonnull final Authentication systemAuthentication + ) { + _entityRegistry = entityRegistry; + _entityClient = entityClient; + _searchService = searchService; + _systemAuthentication = systemAuthentication; + } + + @Value("${siblings.enabled:false}") + private Boolean enabled; + + @VisibleForTesting + void setEnabled(Boolean newValue) { + enabled = newValue; + } + + @Override + public void init() { + } + + @Override + public void invoke(@Nonnull MetadataChangeLog event) { + if (enabled && isEligibleForProcessing(event)) { + + log.info("Urn {} received by Sibling Hook.", event.getEntityUrn()); + + final Urn urn = getUrnFromEvent(event); + + DatasetUrn datasetUrn = null; + try { + datasetUrn = DatasetUrn.createFromUrn(urn); + } catch (URISyntaxException e) { + log.error("Error while parsing urn {} : {}", event.getEntityUrn(), e.toString()); + throw new RuntimeException("Failed to parse entity urn, skipping processing.", e); + } + + // if we are seeing the key, this means the entity may have been deleted and re-ingested + // in this case we want to re-create its siblings aspects + if (event.getAspectName().equals(DATASET_KEY_ASPECT_NAME)) { + handleEntityKeyEvent(datasetUrn); + } else if (datasetUrn.getPlatformEntity().getPlatformNameEntity().equals(DBT_PLATFORM_NAME)) { + handleDbtDatasetEvent(event, datasetUrn); + } else { + handleSourceDatasetEvent(event, datasetUrn); + } + } + } + + private void handleEntityKeyEvent(DatasetUrn datasetUrn) { + Filter entitiesWithYouAsSiblingFilter = createFilterForEntitiesWithYouAsSibling(datasetUrn); + final SearchResult searchResult = _searchService.search( + "dataset", + "*", + entitiesWithYouAsSiblingFilter, + null, + 0, + 10); + + // we have a match of an entity with you as a sibling, associate yourself back + searchResult.getEntities().forEach(entity -> { + if (!entity.getEntity().equals(datasetUrn)) { + if (datasetUrn.getPlatformEntity().getPlatformNameEntity().equals(DBT_PLATFORM_NAME)) { + setSiblingsAndSoftDeleteSibling(datasetUrn, searchResult.getEntities().get(0).getEntity()); + } else { + setSiblingsAndSoftDeleteSibling(searchResult.getEntities().get(0).getEntity(), datasetUrn); + } + } + }); + } + + // If the upstream is a single source system node & subtype is source, then associate the upstream as your sibling + private void handleDbtDatasetEvent(MetadataChangeLog event, DatasetUrn datasetUrn) { + // we need both UpstreamLineage & Subtypes to determine whether to associate + UpstreamLineage upstreamLineage = null; + SubTypes subTypesAspectOfEntity = null; + + if (event.getAspectName().equals(UPSTREAM_LINEAGE_ASPECT_NAME)) { + upstreamLineage = getUpstreamLineageFromEvent(event); + subTypesAspectOfEntity = getSubtypesFromEntityClient(datasetUrn); + } + + if (event.getAspectName().equals(SUB_TYPES_ASPECT_NAME)) { + subTypesAspectOfEntity = getSubtypesFromEvent(event); + upstreamLineage = getUpstreamLineageFromEntityClient(datasetUrn); + } + + if ( + upstreamLineage != null + && subTypesAspectOfEntity != null + && upstreamLineage.hasUpstreams() + && subTypesAspectOfEntity.hasTypeNames() + && subTypesAspectOfEntity.getTypeNames().contains(SOURCE_SUBTYPE) + ) { + UpstreamArray upstreams = upstreamLineage.getUpstreams(); + if ( + upstreams.size() == 1 + && !upstreams.get(0).getDataset().getPlatformEntity().getPlatformNameEntity().equals(DBT_PLATFORM_NAME)) { + setSiblingsAndSoftDeleteSibling(datasetUrn, upstreams.get(0).getDataset()); + } + } + } + + // if the dataset is not dbt--- it may be produced by a dbt dataset. If so, associate them as siblings + private void handleSourceDatasetEvent(MetadataChangeLog event, DatasetUrn sourceUrn) { + if (event.getAspectName().equals(UPSTREAM_LINEAGE_ASPECT_NAME)) { + UpstreamLineage upstreamLineage = getUpstreamLineageFromEvent(event); + if (upstreamLineage != null && upstreamLineage.hasUpstreams()) { + UpstreamArray upstreams = upstreamLineage.getUpstreams(); + if ( + upstreams.size() == 1 + && upstreams.get(0).getDataset().getPlatformEntity().getPlatformNameEntity().equals(DBT_PLATFORM_NAME)) { + setSiblingsAndSoftDeleteSibling(upstreams.get(0).getDataset(), sourceUrn); + } + } + } + } + + private void setSiblingsAndSoftDeleteSibling(Urn dbtUrn, Urn sourceUrn) { + Siblings existingDbtSiblingAspect = getSiblingsFromEntityClient(dbtUrn); + Siblings existingSourceSiblingAspect = getSiblingsFromEntityClient(sourceUrn); + + log.info("Associating {} and {} as siblings.", dbtUrn.toString(), sourceUrn.toString()); + + if ( + existingDbtSiblingAspect != null + && existingSourceSiblingAspect != null + && existingDbtSiblingAspect.getSiblings().contains(sourceUrn.toString()) + && existingDbtSiblingAspect.getSiblings().contains(dbtUrn.toString()) + ) { + // we have already connected them- we can abort here + return; + } + + AuditStamp auditStamp = getAuditStamp(); + + // set source as a sibling of dbt + Siblings dbtSiblingAspect = new Siblings(); + dbtSiblingAspect.setSiblings(new UrnArray(ImmutableList.of(sourceUrn))); + dbtSiblingAspect.setPrimary(true); + + MetadataChangeProposal dbtSiblingProposal = new MetadataChangeProposal(); + GenericAspect dbtSiblingAspectSerialized = GenericRecordUtils.serializeAspect(dbtSiblingAspect); + + dbtSiblingProposal.setAspect(dbtSiblingAspectSerialized); + dbtSiblingProposal.setAspectName(SIBLINGS_ASPECT_NAME); + dbtSiblingProposal.setEntityType(DATASET_ENTITY_NAME); + dbtSiblingProposal.setChangeType(ChangeType.UPSERT); + dbtSiblingProposal.setEntityUrn(dbtUrn); + + try { + _entityClient.ingestProposal(dbtSiblingProposal, _systemAuthentication); + } catch (RemoteInvocationException e) { + log.error("Error while associating {} with {}: {}", dbtUrn.toString(), sourceUrn.toString(), e.toString()); + throw new RuntimeException("Error ingesting sibling proposal. Skipping processing.", e); + } + + // set dbt as a sibling of source + + Siblings sourceSiblingAspect = new Siblings(); + if (existingSourceSiblingAspect != null) { + sourceSiblingAspect = existingSourceSiblingAspect; + } + + UrnArray newSiblingsUrnArray = + sourceSiblingAspect.hasSiblings() ? sourceSiblingAspect.getSiblings() : new UrnArray(); + if (!newSiblingsUrnArray.contains(dbtUrn)) { + newSiblingsUrnArray.add(dbtUrn); + } + + // clean up any references to stale siblings that have been deleted + List filteredNewSiblingsArray = + newSiblingsUrnArray.stream().filter(urn -> { + try { + return _entityClient.exists(urn, _systemAuthentication); + } catch (RemoteInvocationException e) { + log.error("Error while checking existence of {}: {}", urn.toString(), e.toString()); + throw new RuntimeException("Error checking existence. Skipping processing.", e); + } + }).collect(Collectors.toList()); + + sourceSiblingAspect.setSiblings(new UrnArray(filteredNewSiblingsArray)); + sourceSiblingAspect.setPrimary(false); + + MetadataChangeProposal sourceSiblingProposal = new MetadataChangeProposal(); + GenericAspect sourceSiblingAspectSerialized = GenericRecordUtils.serializeAspect(sourceSiblingAspect); + + sourceSiblingProposal.setAspect(sourceSiblingAspectSerialized); + sourceSiblingProposal.setAspectName(SIBLINGS_ASPECT_NAME); + sourceSiblingProposal.setEntityType(DATASET_ENTITY_NAME); + sourceSiblingProposal.setChangeType(ChangeType.UPSERT); + sourceSiblingProposal.setEntityUrn(sourceUrn); + + try { + _entityClient.ingestProposal(sourceSiblingProposal, _systemAuthentication); + } catch (RemoteInvocationException e) { + log.error("Error while associating {} with {}: {}", dbtUrn.toString(), sourceUrn.toString(), e.toString()); + throw new RuntimeException("Error ingesting sibling proposal. Skipping processing.", e); + } + } + + + /** + * Returns true if the event should be processed, which is only true if the event represents a dataset for now + */ + private boolean isEligibleForProcessing(final MetadataChangeLog event) { + return event.getEntityType().equals("dataset") + && !event.getChangeType().equals(ChangeType.DELETE) + && ( + event.getAspectName().equals(UPSTREAM_LINEAGE_ASPECT_NAME) + || event.getAspectName().equals(SUB_TYPES_ASPECT_NAME) + || event.getAspectName().equals(DATASET_KEY_ASPECT_NAME) + ); + } + + /** + * Extracts and returns an {@link Urn} from a {@link MetadataChangeLog}. Extracts from either an entityUrn + * or entityKey field, depending on which is present. + */ + private Urn getUrnFromEvent(final MetadataChangeLog event) { + EntitySpec entitySpec; + try { + entitySpec = _entityRegistry.getEntitySpec(event.getEntityType()); + } catch (IllegalArgumentException e) { + log.error("Error while processing entity type {}: {}", event.getEntityType(), e.toString()); + throw new RuntimeException("Failed to get urn from MetadataChangeLog event. Skipping processing.", e); + } + // Extract an URN from the Log Event. + return EntityKeyUtils.getUrnFromLog(event, entitySpec.getKeyAspectSpec()); + } + + /** + * Deserializes and returns an instance of {@link UpstreamLineage} extracted from a {@link MetadataChangeLog} event. + */ + private UpstreamLineage getUpstreamLineageFromEvent(final MetadataChangeLog event) { + EntitySpec entitySpec; + if (!event.getAspectName().equals(UPSTREAM_LINEAGE_ASPECT_NAME)) { + return null; + } + + try { + entitySpec = _entityRegistry.getEntitySpec(event.getEntityType()); + } catch (IllegalArgumentException e) { + log.error("Error while processing entity type {}: {}", event.getEntityType(), e.toString()); + throw new RuntimeException("Failed to get UpstreamLineage from MetadataChangeLog event. Skipping processing.", e); + } + return (UpstreamLineage) GenericRecordUtils.deserializeAspect( + event.getAspect().getValue(), + event.getAspect().getContentType(), + entitySpec.getAspectSpec(UPSTREAM_LINEAGE_ASPECT_NAME)); + } + + /** + * Deserializes and returns an instance of {@link SubTypes} extracted from a {@link MetadataChangeLog} event. + */ + private SubTypes getSubtypesFromEvent(final MetadataChangeLog event) { + EntitySpec entitySpec; + if (!event.getAspectName().equals(SUB_TYPES_ASPECT_NAME)) { + return null; + } + + try { + entitySpec = _entityRegistry.getEntitySpec(event.getEntityType()); + } catch (IllegalArgumentException e) { + log.error("Error while processing entity type {}: {}", event.getEntityType(), e.toString()); + throw new RuntimeException("Failed to get SubTypes from MetadataChangeLog event. Skipping processing.", e); + } + return (SubTypes) GenericRecordUtils.deserializeAspect( + event.getAspect().getValue(), + event.getAspect().getContentType(), + entitySpec.getAspectSpec(SUB_TYPES_ASPECT_NAME)); + } + + @SneakyThrows + private AuditStamp getAuditStamp() { + return new AuditStamp().setActor(Urn.createFromString(SIBLING_ASSOCIATION_SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + } + + private Filter createFilterForEntitiesWithYouAsSibling( + final Urn entityUrn + ) { + final Filter filter = new Filter(); + final ConjunctiveCriterionArray disjunction = new ConjunctiveCriterionArray(); + + final ConjunctiveCriterion conjunction = new ConjunctiveCriterion(); + final CriterionArray andCriterion = new CriterionArray(); + + final Criterion urnCriterion = new Criterion(); + urnCriterion.setField("siblings.keyword"); + urnCriterion.setValue(entityUrn.toString()); + urnCriterion.setCondition(Condition.EQUAL); + andCriterion.add(urnCriterion); + + conjunction.setAnd(andCriterion); + + disjunction.add(conjunction); + + filter.setOr(disjunction); + return filter; + } + + private SubTypes getSubtypesFromEntityClient( + final Urn urn + ) { + try { + EntityResponse entityResponse = _entityClient.getV2( + DATASET_ENTITY_NAME, + urn, + ImmutableSet.of(SUB_TYPES_ASPECT_NAME), + _systemAuthentication + ); + + if (entityResponse != null && entityResponse.hasAspects() && entityResponse.getAspects().containsKey(Constants.SUB_TYPES_ASPECT_NAME)) { + return new SubTypes(entityResponse.getAspects().get(Constants.SUB_TYPES_ASPECT_NAME).getValue().data()); + } else { + return null; + } + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException("Failed to retrieve Subtypes", e); + } + } + + private UpstreamLineage getUpstreamLineageFromEntityClient( + final Urn urn + ) { + try { + EntityResponse entityResponse = _entityClient.getV2( + DATASET_ENTITY_NAME, + urn, + ImmutableSet.of(UPSTREAM_LINEAGE_ASPECT_NAME), + _systemAuthentication + ); + + if (entityResponse != null && entityResponse.hasAspects() && entityResponse.getAspects().containsKey(Constants.UPSTREAM_LINEAGE_ASPECT_NAME)) { + return new UpstreamLineage(entityResponse.getAspects().get(Constants.UPSTREAM_LINEAGE_ASPECT_NAME).getValue().data()); + } else { + return null; + } + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException("Failed to retrieve UpstreamLineage", e); + } + } + + private Siblings getSiblingsFromEntityClient( + final Urn urn + ) { + try { + EntityResponse entityResponse = _entityClient.getV2( + DATASET_ENTITY_NAME, + urn, + ImmutableSet.of(SIBLINGS_ASPECT_NAME), + _systemAuthentication + ); + + if (entityResponse != null && entityResponse.hasAspects() && entityResponse.getAspects().containsKey(Constants.SIBLINGS_ASPECT_NAME)) { + return new Siblings(entityResponse.getAspects().get(Constants.SIBLINGS_ASPECT_NAME).getValue().data()); + } else { + return null; + } + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException("Failed to retrieve UpstreamLineage", e); + } + } + +} diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java new file mode 100644 index 00000000000000..3ab8175115646d --- /dev/null +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java @@ -0,0 +1,304 @@ +package com.linkedin.metadata.kafka.hook.siblings; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.FabricType; +import com.linkedin.common.Siblings; +import com.linkedin.common.SubTypes; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; +import com.linkedin.dataset.DatasetLineageType; +import com.linkedin.dataset.Upstream; +import com.linkedin.dataset.UpstreamArray; +import com.linkedin.dataset.UpstreamLineage; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.key.DatasetKey; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeLog; +import com.linkedin.mxe.MetadataChangeProposal; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.ArgumentMatchers.*; + + +public class SiblingAssociationHookTest { + private SiblingAssociationHook _siblingAssociationHook; + RestliEntityClient _mockEntityClient; + EntitySearchService _mockSearchService; + Authentication _mockAuthentication; + + @BeforeMethod + public void setupTest() { + EntityRegistry registry = new ConfigEntityRegistry( + SiblingAssociationHookTest.class.getClassLoader().getResourceAsStream("test-entity-registry-siblings.yml")); + _mockEntityClient = Mockito.mock(RestliEntityClient.class); + _mockSearchService = Mockito.mock(EntitySearchService.class); + _mockAuthentication = Mockito.mock(Authentication.class); + _siblingAssociationHook = new SiblingAssociationHook(registry, _mockEntityClient, _mockSearchService, _mockAuthentication); + _siblingAssociationHook.setEnabled(true); + } + + @Test + public void testInvokeWhenThereIsAPairWithDbtSourceNode() throws Exception { + SubTypes mockSourceSubtypesAspect = new SubTypes(); + mockSourceSubtypesAspect.setTypeNames(new StringArray(ImmutableList.of("source"))); + EnvelopedAspectMap mockResponseMap = new EnvelopedAspectMap(); + mockResponseMap.put(SUB_TYPES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(mockSourceSubtypesAspect.data()))); + EntityResponse mockResponse = new EntityResponse(); + mockResponse.setAspects(mockResponseMap); + + Mockito.when(_mockEntityClient.exists(Mockito.any(), Mockito.any())).thenReturn(true); + + + Mockito.when( + _mockEntityClient.getV2( + DATASET_ENTITY_NAME, + Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)"), + ImmutableSet.of(SUB_TYPES_ASPECT_NAME), + _mockAuthentication + )).thenReturn(mockResponse); + + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + final UpstreamLineage upstreamLineage = new UpstreamLineage(); + final UpstreamArray upstreamArray = new UpstreamArray(); + final Upstream upstream = new Upstream(); + upstream.setType(DatasetLineageType.TRANSFORMED); + upstream.setDataset(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + + upstreamArray.add(upstream); + upstreamLineage.setUpstreams(upstreamArray); + + event.setAspect(GenericRecordUtils.serializeAspect(upstreamLineage)); + event.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + _siblingAssociationHook.invoke(event); + + final Siblings dbtSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(true); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + proposal.setEntityType(DATASET_ENTITY_NAME); + proposal.setAspectName(SIBLINGS_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(dbtSiblingsAspect)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.eq(_mockAuthentication) + ); + + final Siblings sourceSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(false); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + proposal2.setEntityType(DATASET_ENTITY_NAME); + proposal2.setAspectName(SIBLINGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(sourceSiblingsAspect)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.eq(_mockAuthentication) + ); + } + + @Test + public void testInvokeWhenThereIsNoPairWithDbtModel() throws Exception { + SubTypes mockSourceSubtypesAspect = new SubTypes(); + mockSourceSubtypesAspect.setTypeNames(new StringArray(ImmutableList.of("model"))); + + Mockito.when(_mockEntityClient.exists(Mockito.any(), Mockito.any())).thenReturn(true); + + EnvelopedAspectMap mockResponseMap = new EnvelopedAspectMap(); + mockResponseMap.put(SUB_TYPES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(mockSourceSubtypesAspect.data()))); + EntityResponse mockResponse = new EntityResponse(); + mockResponse.setAspects(mockResponseMap); + + Mockito.when(_mockEntityClient.exists(Mockito.any(), Mockito.any())).thenReturn(true); + + + Mockito.when( + _mockEntityClient.getV2( + DATASET_ENTITY_NAME, + Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)"), + ImmutableSet.of(SUB_TYPES_ASPECT_NAME), + _mockAuthentication + )).thenReturn(mockResponse); + + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + final UpstreamLineage upstreamLineage = new UpstreamLineage(); + final UpstreamArray upstreamArray = new UpstreamArray(); + final Upstream upstream = new Upstream(); + upstream.setType(DatasetLineageType.TRANSFORMED); + upstream.setDataset(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + + upstreamArray.add(upstream); + upstreamLineage.setUpstreams(upstreamArray); + + event.setAspect(GenericRecordUtils.serializeAspect(upstreamLineage)); + event.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + _siblingAssociationHook.invoke(event); + + final Siblings dbtSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(true); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + proposal.setEntityType(DATASET_ENTITY_NAME); + proposal.setAspectName(SIBLINGS_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(dbtSiblingsAspect)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(0)).ingestProposal( + Mockito.eq(proposal), + Mockito.eq(_mockAuthentication) + ); + } + + @Test + public void testInvokeWhenThereIsAPairWithBigqueryDownstreamNode() throws Exception { + Mockito.when(_mockEntityClient.exists(Mockito.any(), Mockito.any())).thenReturn(true); + + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(UPSTREAM_LINEAGE_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + final UpstreamLineage upstreamLineage = new UpstreamLineage(); + final UpstreamArray upstreamArray = new UpstreamArray(); + final Upstream upstream = new Upstream(); + upstream.setType(DatasetLineageType.TRANSFORMED); + upstream.setDataset(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + + upstreamArray.add(upstream); + upstreamLineage.setUpstreams(upstreamArray); + + event.setAspect(GenericRecordUtils.serializeAspect(upstreamLineage)); + event.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + _siblingAssociationHook.invoke(event); + + final Siblings dbtSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(true); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + proposal.setEntityType(DATASET_ENTITY_NAME); + proposal.setAspectName(SIBLINGS_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(dbtSiblingsAspect)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.eq(_mockAuthentication) + ); + + final Siblings sourceSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(false); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + proposal2.setEntityType(DATASET_ENTITY_NAME); + proposal2.setAspectName(SIBLINGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(sourceSiblingsAspect)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.eq(_mockAuthentication) + ); + } + + @Test + public void testInvokeWhenThereIsAKeyBeingReingested() throws Exception { + Mockito.when(_mockEntityClient.exists(Mockito.any(), Mockito.any())).thenReturn(true); + + SearchResult returnSearchResult = new SearchResult(); + SearchEntityArray returnEntityArray = new SearchEntityArray(); + SearchEntity returnArrayValue = new SearchEntity(); + returnArrayValue.setEntity( + Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)") + ); + returnEntityArray.add(returnArrayValue); + + returnSearchResult.setEntities(returnEntityArray); + + Mockito.when( + _mockSearchService.search( + anyString(), anyString(), any(), any(), anyInt(), anyInt() + )).thenReturn(returnSearchResult); + + MetadataChangeLog event = new MetadataChangeLog(); + event.setEntityType(DATASET_ENTITY_NAME); + event.setAspectName(DATASET_KEY_ASPECT_NAME); + event.setChangeType(ChangeType.UPSERT); + final DatasetKey datasetKey = new DatasetKey(); + datasetKey.setName("my-proj.jaffle_shop.customers"); + datasetKey.setOrigin(FabricType.PROD); + datasetKey.setPlatform(DataPlatformUrn.createFromString("urn:li:dataPlatform:bigquery")); + + event.setAspect(GenericRecordUtils.serializeAspect(datasetKey)); + event.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + _siblingAssociationHook.invoke(event); + + final Siblings dbtSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(true); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")); + proposal.setEntityType(DATASET_ENTITY_NAME); + proposal.setAspectName(SIBLINGS_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(dbtSiblingsAspect)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal), + Mockito.eq(_mockAuthentication) + ); + + final Siblings sourceSiblingsAspect = new Siblings() + .setSiblings(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:dbt,my-proj.jaffle_shop.customers,PROD)")))) + .setPrimary(false); + + final MetadataChangeProposal proposal2 = new MetadataChangeProposal(); + proposal2.setEntityUrn(Urn.createFromString("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-proj.jaffle_shop.customers,PROD)")); + proposal2.setEntityType(DATASET_ENTITY_NAME); + proposal2.setAspectName(SIBLINGS_ASPECT_NAME); + proposal2.setAspect(GenericRecordUtils.serializeAspect(sourceSiblingsAspect)); + proposal2.setChangeType(ChangeType.UPSERT); + + Mockito.verify(_mockEntityClient, Mockito.times(1)).ingestProposal( + Mockito.eq(proposal2), + Mockito.eq(_mockAuthentication) + ); + } +} diff --git a/metadata-jobs/mce-consumer-job/README.md b/metadata-jobs/mce-consumer-job/README.md index ab622d73b7b229..5d911feef76c02 100644 --- a/metadata-jobs/mce-consumer-job/README.md +++ b/metadata-jobs/mce-consumer-job/README.md @@ -2,34 +2,48 @@ title: "metadata-jobs:mce-consumer-job" --- -# MetadataChangeEvent (MCE) Consumer Job -MCE Consumer is a [Kafka Streams](https://kafka.apache.org/documentation/streams/) job. Its main function is to listen -`MetadataChangeEvent` Kafka topic for messages and process those messages and writes new metadata to `DataHub GMS`. -After every successful update of metadata, GMS fires a `MetadataAuditEvent` and this is consumed by -[MAE Consumer Job](../mae-consumer-job). +# Metadata Change Event Consumer Job + +The Metadata Change Event Consumer is a [Kafka Streams](https://kafka.apache.org/documentation/streams/) job which can be deployed by itself, or as part of the Metadata Service. + +Its main function is to listen to change proposal events emitted by clients of DataHub which request changes to the Metadata Graph. It then applies +these requests against DataHub's storage layer: the Metadata Service. + +Today the job consumes from two topics: + +1. `MetadataChangeProposal_v1` +2. (Deprecated) `MetadataChangeEvent_v4` + +and produces to the following topics + +1. `FailedMetadataChangeProposal_v1` +2. (Deprecated) `FailedMetadataChangeEvent_v4` + +> Where does the misleeading name **Metadata Change Event** come from? Well, history. Previously, this job consumed +> a single `MetadataChangeEvent` topic which has been deprecated and replaced by per-aspect Metadata Change Proposals. Hence, the name! ## Pre-requisites * You need to have [JDK8](https://www.oracle.com/java/technologies/jdk8-downloads.html) installed on your machine to be -able to build `DataHub GMS`. +able to build `DataHub Metadata Service`. ## Build -`MCE Consumer Job` is already built as part of top level build: +`Metadata Change Event Consumer Job` is already built as part of top level build: ``` ./gradlew build ``` -However, if you only want to build `MCE Consumer Job` specifically: +However, if you only want to build `Metadata Change Event Consumer Job` specifically: ``` ./gradlew :metadata-jobs:mce-consumer-job:build ``` ## Dependencies -Before starting `MCE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup) +Before starting `Metadata Change Event Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup) and [DataHub GMS](../../docker/datahub-gms) Docker containers are up and running. ## Start via Docker image -Quickest way to try out `MCE Consumer Job` is running the [Docker image](../../docker/datahub-mce-consumer). +Quickest way to try out `Metadata Change Event Consumer Job` is running the [Docker image](../../docker/datahub-mce-consumer). ## Start via command line If you do modify things and want to try it out quickly without building the Docker image, you can also run diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index cb2e6249acf2d1..ee56e4eaf15b6f 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -50,11 +50,11 @@ public class MetadataChangeEventsProcessor { private final Histogram kafkaLagStats = MetricUtils.get().histogram(MetricRegistry.name(this.getClass(), "kafkaLag")); - @Value("${KAFKA_FMCE_TOPIC_NAME:" + Topics.FAILED_METADATA_CHANGE_EVENT + "}") + @Value("${FAILED_METADATA_CHANGE_EVENT_NAME:${KAFKA_FMCE_TOPIC_NAME:" + Topics.FAILED_METADATA_CHANGE_EVENT + "}}") private String fmceTopicName; @KafkaListener(id = "${METADATA_CHANGE_EVENT_KAFKA_CONSUMER_GROUP_ID:mce-consumer-job-client}", topics = - "${KAFKA_MCE_TOPIC_NAME:" + Topics.METADATA_CHANGE_EVENT + "}", containerFactory = "kafkaEventConsumer") + "${METADATA_CHANGE_EVENT_NAME:${KAFKA_MCE_TOPIC_NAME:" + Topics.METADATA_CHANGE_EVENT + "}}", containerFactory = "kafkaEventConsumer") public void consume(final ConsumerRecord consumerRecord) { kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp()); final GenericRecord record = consumerRecord.value(); diff --git a/metadata-models/src/main/pegasus/com/linkedin/chart/ChartUsageStatistics.pdl b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartUsageStatistics.pdl new file mode 100644 index 00000000000000..f5176add976792 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartUsageStatistics.pdl @@ -0,0 +1,33 @@ +namespace com.linkedin.chart + +import com.linkedin.timeseries.TimeseriesAspectBase + +/** + * Experimental (Subject to breaking change) -- Stats corresponding to chart's usage. + * + * If this aspect represents the latest snapshot of the statistics about a Chart, the eventGranularity field should be null. + * If this aspect represents a bucketed window of usage statistics (e.g. over a day), then the eventGranularity field should be set accordingly. + */ +@Aspect = { + "name": "chartUsageStatistics", + "type": "timeseries", +} +record ChartUsageStatistics includes TimeseriesAspectBase { + /** + * The total number of times chart has been viewed + */ + @TimeseriesField = {} + viewsCount: optional int + + /** + * Unique user count + */ + @TimeseriesField = {} + uniqueUserCount: optional int + + /** + * Users within this bucket, with frequency counts + */ + @TimeseriesFieldCollection = {"key":"user"} + userCounts: optional array[ChartUserUsageCounts] +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/chart/ChartUserUsageCounts.pdl b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartUserUsageCounts.pdl new file mode 100644 index 00000000000000..238ae3c628c30c --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/chart/ChartUserUsageCounts.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.chart + +import com.linkedin.common.Urn + +/** + * Records a single user's usage counts for a given resource + */ +record ChartUserUsageCounts { + /** + * The unique id of the user. + */ + user: Urn + + /** + * The number of times the user has viewed the chart + */ + @TimeseriesField = {} + viewsCount: optional int +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Origin.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Origin.pdl new file mode 100644 index 00000000000000..d65d692a74bd5c --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Origin.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.common + +/** + * Carries information about where an entity originated from. + */ +@Aspect = { + "name": "origin" +} +record Origin { + /** + * Where an entity originated from. Either NATIVE or EXTERNAL. + */ + type: OriginType + + /** + * Only populated if type is EXTERNAL. The externalType of the entity, such as the name of the identity provider. + */ + externalType: optional string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/OriginType.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/OriginType.pdl new file mode 100644 index 00000000000000..008360e2dd05dd --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/OriginType.pdl @@ -0,0 +1,15 @@ +namespace com.linkedin.common + +/** + * Enum to define where an entity originated from. + */ +enum OriginType { + /** + * The entity is native to DataHub. + */ + NATIVE + /** + * The entity is external to DataHub. + */ + EXTERNAL +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Siblings.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Siblings.pdl new file mode 100644 index 00000000000000..06bf36fbd59cce --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Siblings.pdl @@ -0,0 +1,33 @@ +namespace com.linkedin.common + +/** + * Siblings information of an entity. + */ +@Aspect = { + "name": "siblings" +} +record Siblings { + + /** + * List of sibling entities + */ + @Relationship = { + "/*": { + "name": "SiblingOf", + "entityTypes": [ "dataset" ] + } + } + @Searchable = { + "/*": { + "fieldName": "siblings", + "fieldType": "URN", + "queryByDefault": false, + } + } + siblings: array[Urn] + + /** + * If this is the leader entity of the set of siblings + */ + primary: boolean +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl index a41eab88be7227..7a5499382e4e08 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl @@ -33,5 +33,9 @@ record ContainerProperties includes CustomProperties, ExternalReference { /** * Description of the Asset Container as it exists inside a source system */ + @Searchable = { + "fieldType": "TEXT", + "hasValuesFieldName": "hasDescription" + } description: optional string } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/container/EditableContainerProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/container/EditableContainerProperties.pdl index 1e7fd803e10349..718c1210452c4a 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/container/EditableContainerProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/container/EditableContainerProperties.pdl @@ -10,5 +10,9 @@ record EditableContainerProperties { /** * Description of the Asset Container as its received on the DataHub Platform */ + @Searchable = { + "fieldType": "TEXT", + "fieldName": "editedDescription", + } description: optional string } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl index 10549227213c42..bf498f5af62445 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardInfo.pdl @@ -5,6 +5,7 @@ import com.linkedin.common.ChangeAuditStamps import com.linkedin.common.ChartUrn import com.linkedin.common.Time import com.linkedin.common.Url +import com.linkedin.common.Urn import com.linkedin.common.CustomProperties import com.linkedin.common.ExternalReference @@ -47,6 +48,18 @@ record DashboardInfo includes CustomProperties, ExternalReference { } charts: array[ChartUrn] = [ ] + /** + * Datasets consumed by a dashboard + */ + @Relationship = { + "/*": { + "name": "Consumes", + "entityTypes": [ "dataset" ], + "isLineage": true + } + } + datasets: array[Urn] = [ ] + /** * Captures information about who created/last modified/deleted this dashboard and when */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardUsageStatistics.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardUsageStatistics.pdl new file mode 100644 index 00000000000000..b45989847c6748 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardUsageStatistics.pdl @@ -0,0 +1,54 @@ +namespace com.linkedin.dashboard + +import com.linkedin.timeseries.TimeseriesAspectBase + +/** + * Experimental (Subject to breaking change) -- Stats corresponding to dashboard's usage. + * + * If this aspect represents the latest snapshot of the statistics about a Dashboard, the eventGranularity field should be null. + * If this aspect represents a bucketed window of usage statistics (e.g. over a day), then the eventGranularity field should be set accordingly. + */ +@Aspect = { + "name": "dashboardUsageStatistics", + "type": "timeseries", +} +record DashboardUsageStatistics includes TimeseriesAspectBase { + /** + * The total number of times dashboard has been viewed + */ + @TimeseriesField = {} + viewsCount: optional int + + /** + * The total number of dashboard executions (refreshes / syncs) + */ + @TimeseriesField = {} + executionsCount: optional int + + /** + * Unique user count + */ + @TimeseriesField = {} + uniqueUserCount: optional int + + /** + * Users within this bucket, with frequency counts + */ + @TimeseriesFieldCollection = {"key":"user"} + userCounts: optional array[DashboardUserUsageCounts] + + /** + * The total number of times that the dashboard has been favorited + */ + @TimeseriesField = {} + favoritesCount: optional int + + /** + * Last viewed at + * + * This should not be set in cases where statistics are windowed. + */ + @TimeseriesField = {} + lastViewedAt: optional long + +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardUserUsageCounts.pdl b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardUserUsageCounts.pdl new file mode 100644 index 00000000000000..78119878874704 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/dashboard/DashboardUserUsageCounts.pdl @@ -0,0 +1,38 @@ +namespace com.linkedin.dashboard + +import com.linkedin.common.Urn + +/** + * Records a single user's usage counts for a given resource + */ +record DashboardUserUsageCounts { + /** + * The unique id of the user. + */ + user: Urn + + /** + * The number of times the user has viewed the dashboard + */ + @TimeseriesField = {} + viewsCount: optional int + + /** + * The number of times the user has executed (refreshed) the dashboard + */ + @TimeseriesField = {} + executionsCount: optional int + + /** + * Normalized numeric metric representing user's dashboard usage -- the number of times the user executed or viewed the dashboard. + */ + @TimeseriesField = {} + usageCount: optional int + + /** + * If user_email is set, we attempt to resolve the user's urn upon ingest + */ + @TimeseriesField = {} + userEmail: optional string + +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl index 28e73c38b01728..3942bf4262e517 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/entity/EnvelopedAspect.pdl @@ -37,4 +37,9 @@ record EnvelopedAspect { * The audit stamp detailing who the aspect was created by and when **/ created: AuditStamp + + /** + * The system metadata for this aspect + **/ + systemMetadata: optional SystemMetadata } diff --git a/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl index f27c13dc172dff..29acd0aa523899 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/execution/ExecutionRequestResult.pdl @@ -17,6 +17,11 @@ record ExecutionRequestResult { */ report: optional string + /** + * A structured report if available. + */ + structuredReport: optional StructuredExecutionReport + /** * Time at which the request was created */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/execution/StructuredExecutionReport.pdl b/metadata-models/src/main/pegasus/com/linkedin/execution/StructuredExecutionReport.pdl new file mode 100644 index 00000000000000..72eb7baf558a23 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/execution/StructuredExecutionReport.pdl @@ -0,0 +1,25 @@ +namespace com.linkedin.execution + +/** + * A flexible carrier for structured results of an execution request. + * The goal is to allow for free flow of structured responses from execution tasks to the orchestrator or observer. + * The full spectrum of different execution report types is not intended to be modeled by this object. + */ + +record StructuredExecutionReport { + + /** + * The type of the structured report. (e.g. INGESTION_REPORT, TEST_CONNECTION_REPORT, etc.) + */ + type: string + + /** + * The serialized value of the structured report + **/ + serializedValue: string + + /** + * The content-type of the serialized value (e.g. application/json, application/json;gzip etc.) + **/ + contentType: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserCredentials.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserCredentials.pdl new file mode 100644 index 00000000000000..9b7c6ac2fbdce1 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserCredentials.pdl @@ -0,0 +1,32 @@ +namespace com.linkedin.identity + +import com.linkedin.common.CorpuserUrn + +/** + * Corp user credentials + */ +@Aspect = { + "name": "corpUserCredentials" +} +@Aspect.EntityUrns = [ "com.linkedin.common.CorpuserUrn" ] +record CorpUserCredentials { + /** + * Salt used to hash password + */ + salt: string + + /** + * Hashed password generated by concatenating salt and password, then hashing + */ + hashedPassword: string + + /** + * Optional token needed to reset a user's password. Can only be set by the admin. + */ + passwordResetToken: optional string + + /** + * When the password reset token expires. + */ + passwordResetTokenExpirationTimeMillis: optional long +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/InviteToken.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/InviteToken.pdl new file mode 100644 index 00000000000000..16559bfc2ccaaa --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/InviteToken.pdl @@ -0,0 +1,16 @@ +namespace com.linkedin.identity + +import com.linkedin.common.Urn + +/** + * Aspect used to store the token needed to invite native DataHub users + */ +@Aspect = { + "name": "inviteToken" +} +record InviteToken { + /** + * The encrypted invite token. + */ + token: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/NativeGroupMembership.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/NativeGroupMembership.pdl new file mode 100644 index 00000000000000..6ec6378e79dd44 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/NativeGroupMembership.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.identity + +import com.linkedin.common.Urn + +/** + * Carries information about the native CorpGroups a user is in. + */ +@Aspect = { + "name": "nativeGroupMembership" +} +record NativeGroupMembership { + @Relationship = { + "/*": { + "name": "IsMemberOfNativeGroup", + "entityTypes": [ "corpGroup" ] + } + } + nativeGroups: array[Urn] +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/InviteTokenKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/InviteTokenKey.pdl new file mode 100644 index 00000000000000..d486a391907d47 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/InviteTokenKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.metadata.key + +/** + * Key for an InviteToken. + */ +@Aspect = { + "name": "inviteTokenKey" +} +record InviteTokenKey { + /** + * A unique id for the invite token. + */ + id: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl index 6f91baf6774920..94eb1c427d9574 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/SearchFlags.pdl @@ -8,4 +8,9 @@ record SearchFlags { * Whether to skip cache */ skipCache: boolean = false + + /** + * The maximum number of values in an facet aggregation + */ + maxAggValues: int = 20 } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/filter/Condition.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/filter/Condition.pdl index 93924f7c66fcc4..35d3a7e823f460 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/filter/Condition.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/filter/Condition.pdl @@ -20,6 +20,11 @@ enum Condition { */ EQUAL + /** + * Represent the relation: field is null, e.g. platform is null + */ + IS_NULL + /** * Represent the relation greater than, e.g. ownerCount > 5 */ diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl index 5f3c90e7f4a593..bf63933ff1f678 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaMetadata.pdl @@ -3,6 +3,7 @@ namespace com.linkedin.schema import com.linkedin.common.ChangeAuditStamps import com.linkedin.common.DatasetUrn import com.linkedin.dataset.SchemaFieldPath +import com.linkedin.mxe.SystemMetadata /** * SchemaMetadata to describe metadata related to store schema diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index c024ddfc37e5fb..78d5d96eeda94e 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -20,6 +20,7 @@ entities: - container - deprecation - testResults + - siblings - name: dataHubPolicy doc: DataHub Policies represent access policies granted to users or groups on metadata operations like edit, view etc. category: internal @@ -62,6 +63,7 @@ entities: - domains - container - deprecation + - dashboardUsageStatistics - name: notebook doc: Notebook represents a combination of query, text, chart and etc. This is in BETA version keyAspect: notebookKey @@ -88,6 +90,8 @@ entities: - groupMembership - globalTags - status + - corpUserCredentials + - nativeGroupMembership - name: corpGroup doc: CorpGroup represents an identity of a group of users in the enterprise. keyAspect: corpGroupKey @@ -97,6 +101,7 @@ entities: - globalTags - ownership - status + - origin - name: domain doc: A data domain within an organization. keyAspect: domainKey @@ -232,4 +237,9 @@ entities: aspects: - dataHubUpgradeRequest - dataHubUpgradeResult + - name: inviteToken + category: core + keyAspect: inviteTokenKey + aspects: + - inviteToken events: diff --git a/metadata-models/src/main/resources/test-entity-registry-siblings.yml b/metadata-models/src/main/resources/test-entity-registry-siblings.yml new file mode 100644 index 00000000000000..421213a95d3adf --- /dev/null +++ b/metadata-models/src/main/resources/test-entity-registry-siblings.yml @@ -0,0 +1,9 @@ +entities: + - name: dataset + keyAspect: datasetKey + aspects: + - status + - siblings + - datasetProperties + - subTypes + - upstreamLineage diff --git a/metadata-service/auth-api/src/main/java/com/datahub/authentication/AuthenticationConstants.java b/metadata-service/auth-api/src/main/java/com/datahub/authentication/AuthenticationConstants.java index 07f9a01d027ca0..95f4dbcc7d8756 100644 --- a/metadata-service/auth-api/src/main/java/com/datahub/authentication/AuthenticationConstants.java +++ b/metadata-service/auth-api/src/main/java/com/datahub/authentication/AuthenticationConstants.java @@ -30,6 +30,8 @@ public class AuthenticationConstants { public static final String SYSTEM_CLIENT_SECRET_CONFIG = "systemClientSecret"; public static final String ENTITY_SERVICE = "entityService"; + public static final String TOKEN_SERVICE = "tokenService"; + private AuthenticationConstants() { } } diff --git a/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java b/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java index 4da62969011fe9..0953296ee20ebe 100644 --- a/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java +++ b/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizedActors.java @@ -2,11 +2,15 @@ import com.linkedin.common.urn.Urn; import java.util.List; -import lombok.Builder; + +import lombok.AccessLevel; import lombok.Value; +import lombok.AllArgsConstructor; +import lombok.Builder; @Value +@AllArgsConstructor(access = AccessLevel.PUBLIC) @Builder public class AuthorizedActors { String privilege; diff --git a/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java b/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java index 321b9ee87a5cf1..b83253cd2af041 100644 --- a/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java +++ b/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizerConfiguration.java @@ -9,6 +9,10 @@ */ @Data public class AuthorizerConfiguration { + /** + * Whether to enable this authorizer + */ + private boolean enabled; /** * A fully-qualified class name for the {@link Authorizer} implementation to be registered. */ @@ -17,4 +21,5 @@ public class AuthorizerConfiguration { * A set of authorizer-specific configurations passed through during "init" of the authorizer. */ private Map configs; + } diff --git a/metadata-service/auth-api/src/main/java/com/datahub/authorization/DefaultAuthorizerConfiguration.java b/metadata-service/auth-api/src/main/java/com/datahub/authorization/DefaultAuthorizerConfiguration.java index fd8375c1620871..dfec06dedd147c 100644 --- a/metadata-service/auth-api/src/main/java/com/datahub/authorization/DefaultAuthorizerConfiguration.java +++ b/metadata-service/auth-api/src/main/java/com/datahub/authorization/DefaultAuthorizerConfiguration.java @@ -2,6 +2,7 @@ import lombok.Data; + @Data public class DefaultAuthorizerConfiguration { /** diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/authentication/filter/AuthenticationFilter.java index 8919bb7bbeda3d..7bae8bcd3dc0aa 100644 --- a/metadata-service/auth-filter/src/main/java/com/datahub/authentication/filter/AuthenticationFilter.java +++ b/metadata-service/auth-filter/src/main/java/com/datahub/authentication/filter/AuthenticationFilter.java @@ -11,6 +11,7 @@ import com.datahub.authentication.authenticator.AuthenticatorChain; import com.datahub.authentication.authenticator.DataHubSystemAuthenticator; import com.datahub.authentication.authenticator.NoOpAuthenticator; +import com.datahub.authentication.token.StatefulTokenService; import com.google.common.collect.ImmutableMap; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.entity.EntityService; @@ -49,6 +50,10 @@ public class AuthenticationFilter implements Filter { @Named("entityService") private EntityService _entityService; + @Inject + @Named("dataHubTokenService") + private StatefulTokenService _tokenService; + private AuthenticatorChain authenticatorChain; @Override @@ -109,8 +114,12 @@ private void buildAuthenticatorChain() { boolean isAuthEnabled = this.configurationProvider.getAuthentication().isEnabled(); // Create authentication context object to pass to authenticator instances. They can use it as needed. - final AuthenticatorContext authenticatorContext = new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, - this._entityService)); + final AuthenticatorContext authenticatorContext = new AuthenticatorContext(ImmutableMap.of( + ENTITY_SERVICE, + this._entityService, + TOKEN_SERVICE, + this._tokenService + )); if (isAuthEnabled) { log.info("Auth is enabled. Building authenticator chain..."); diff --git a/metadata-service/auth-impl/build.gradle b/metadata-service/auth-impl/build.gradle index f87b2cd2b2c892..0887d7199faef2 100644 --- a/metadata-service/auth-impl/build.gradle +++ b/metadata-service/auth-impl/build.gradle @@ -1,9 +1,14 @@ apply plugin: 'java' + +compileJava { + options.debug = true +} + dependencies { compile project(path: ':metadata-models') compile project(path: ':metadata-service:restli-client') - compile project(path: ':metadata-service:auth-api'); + compile project(path: ':metadata-service:auth-api') implementation 'io.jsonwebtoken:jjwt-api:0.11.2' runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.11.2', @@ -15,4 +20,4 @@ dependencies { testCompile externalDependency.mockito -} +} \ No newline at end of file diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticator.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticator.java index 86eba7a13aa2a0..af785c1af0a12e 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticator.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticator.java @@ -62,9 +62,7 @@ public void init(@Nonnull final Map config, final AuthenticatorC "Unable to initialize DataHubTokenAuthenticator, entity service reference is not of type: " + "EntityService.class, found: " + entityService.getClass()); } - this._statefulTokenService = - new StatefulTokenService(signingKey, signingAlgorithm, DEFAULT_ISSUER, (EntityService) entityService, - salt); + this._statefulTokenService = (StatefulTokenService) Objects.requireNonNull(context.data().get(TOKEN_SERVICE)); } @Override diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java new file mode 100644 index 00000000000000..e1ea1f0e6350fa --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java @@ -0,0 +1,251 @@ +package com.datahub.authentication.group; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.CorpGroupUrnArray; +import com.linkedin.common.CorpuserUrnArray; +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.Origin; +import com.linkedin.common.OriginType; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.identity.CorpGroupInfo; +import com.linkedin.identity.GroupMembership; +import com.linkedin.identity.NativeGroupMembership; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.key.CorpGroupKey; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import static com.linkedin.metadata.Constants.*; + + +public class GroupService { + private final EntityClient _entityClient; + private final EntityService _entityService; + private final GraphClient _graphClient; + + public GroupService(@Nonnull EntityClient entityClient, @Nonnull EntityService entityService, + @Nonnull GraphClient graphClient) { + Objects.requireNonNull(entityClient, "entityClient must not be null!"); + Objects.requireNonNull(entityService, "entityService must not be null!"); + Objects.requireNonNull(graphClient, "secretService must not be null!"); + + _entityClient = entityClient; + _entityService = entityService; + _graphClient = graphClient; + } + + public boolean groupExists(@Nonnull Urn groupUrn) { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + return _entityService.exists(groupUrn); + } + + public Origin getGroupOrigin(@Nonnull final Urn groupUrn) { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + return (Origin) _entityService.getLatestAspect(groupUrn, ORIGIN_ASPECT_NAME); + } + + public void addUserToNativeGroup(@Nonnull final Urn userUrn, @Nonnull final Urn groupUrn, + final Authentication authentication) { + Objects.requireNonNull(userUrn, "userUrn must not be null"); + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + + // Verify the user exists + if (!_entityService.exists(userUrn)) { + throw new RuntimeException("Failed to add member to group. User does not exist."); + } + + try { + // First, fetch user's group membership aspect. + NativeGroupMembership nativeGroupMembership = getExistingNativeGroupMembership(userUrn, authentication); + // Handle the duplicate case. + nativeGroupMembership.getNativeGroups().remove(groupUrn); + nativeGroupMembership.getNativeGroups().add(groupUrn); + + // Finally, create the MetadataChangeProposal. + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(userUrn); + proposal.setEntityType(CORP_USER_ENTITY_NAME); + proposal.setAspectName(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(nativeGroupMembership)); + proposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(proposal, authentication); + } catch (Exception e) { + throw new RuntimeException("Failed to add member to group", e); + } + } + + public String createNativeGroup(@Nonnull final CorpGroupKey corpGroupKey, @Nonnull final String groupName, + @Nonnull final String groupDescription, final Authentication authentication) throws Exception { + Objects.requireNonNull(corpGroupKey, "corpGroupKey must not be null"); + Objects.requireNonNull(groupName, "groupName must not be null"); + Objects.requireNonNull(groupDescription, "groupDescription must not be null"); + + Urn corpGroupUrn = EntityKeyUtils.convertEntityKeyToUrn(corpGroupKey, Constants.CORP_GROUP_ENTITY_NAME); + if (groupExists(corpGroupUrn)) { + throw new IllegalArgumentException("This Group already exists!"); + } + + String groupInfo = createGroupInfo(corpGroupKey, groupName, groupDescription, authentication); + createNativeGroupOrigin(corpGroupUrn, authentication); + return groupInfo; + } + + public void removeExistingNativeGroupMembers(@Nonnull final Urn groupUrn, @Nonnull final List userUrnList, + final Authentication authentication) throws Exception { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + Objects.requireNonNull(userUrnList, "userUrnList must not be null"); + + final Set userUrns = new HashSet<>(userUrnList); + for (Urn userUrn : userUrns) { + final Map entityResponseMap = _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, userUrns, + Collections.singleton(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME), authentication); + EntityResponse entityResponse = entityResponseMap.get(userUrn); + if (entityResponse == null) { + continue; + } + + final NativeGroupMembership nativeGroupMembership = new NativeGroupMembership( + entityResponse.getAspects().get(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()); + if (nativeGroupMembership.getNativeGroups().remove(groupUrn)) { + // Finally, create the MetadataChangeProposal. + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(userUrn); + proposal.setEntityType(CORP_USER_ENTITY_NAME); + proposal.setAspectName(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(nativeGroupMembership)); + proposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(proposal, authentication); + } + } + } + + public void migrateGroupMembershipToNativeGroupMembership(@Nonnull final Urn groupUrn, final String actorUrnStr, + final Authentication authentication) throws Exception { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + + // Get the existing set of users + final List userUrnList = getExistingGroupMembers(groupUrn, actorUrnStr); + // Remove the existing group membership for each user in the group + removeExistingGroupMembers(groupUrn, userUrnList, authentication); + // Mark the group as a native group + createNativeGroupOrigin(groupUrn, authentication); + // Add each user as a native group member to the group + userUrnList.forEach(userUrn -> addUserToNativeGroup(userUrn, groupUrn, authentication)); + } + + NativeGroupMembership getExistingNativeGroupMembership(@Nonnull final Urn userUrn, + final Authentication authentication) throws Exception { + final EntityResponse entityResponse = + _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, Collections.singleton(userUrn), + Collections.singleton(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME), authentication).get(userUrn); + + NativeGroupMembership nativeGroupMembership; + if (entityResponse == null || !entityResponse.getAspects().containsKey(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)) { + // If the user doesn't have the NativeGroupMembership aspect, create one. + nativeGroupMembership = new NativeGroupMembership(); + nativeGroupMembership.setNativeGroups(new UrnArray()); + } else { + nativeGroupMembership = new NativeGroupMembership( + entityResponse.getAspects().get(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()); + } + return nativeGroupMembership; + } + + String createGroupInfo(@Nonnull final CorpGroupKey corpGroupKey, @Nonnull final String groupName, + @Nonnull final String groupDescription, final Authentication authentication) throws Exception { + Objects.requireNonNull(corpGroupKey, "corpGroupKey must not be null"); + Objects.requireNonNull(groupName, "groupName must not be null"); + Objects.requireNonNull(groupDescription, "groupDescription must not be null"); + + // Create the Group info. + final CorpGroupInfo corpGroupInfo = new CorpGroupInfo(); + corpGroupInfo.setDisplayName(groupName); + corpGroupInfo.setDescription(groupDescription); + corpGroupInfo.setGroups(new CorpGroupUrnArray()); + corpGroupInfo.setMembers(new CorpuserUrnArray()); + corpGroupInfo.setAdmins(new CorpuserUrnArray()); + + // Finally, create the MetadataChangeProposal. + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(corpGroupKey)); + proposal.setEntityType(Constants.CORP_GROUP_ENTITY_NAME); + proposal.setAspectName(Constants.CORP_GROUP_INFO_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(corpGroupInfo)); + proposal.setChangeType(ChangeType.UPSERT); + return _entityClient.ingestProposal(proposal, authentication); + } + + void createNativeGroupOrigin(@Nonnull final Urn groupUrn, final Authentication authentication) throws Exception { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + + // Create the Group info. + final Origin groupOrigin = new Origin(); + groupOrigin.setType(OriginType.NATIVE); + + // Finally, create the MetadataChangeProposal. + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(groupUrn); + proposal.setEntityType(Constants.CORP_GROUP_ENTITY_NAME); + proposal.setAspectName(ORIGIN_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(groupOrigin)); + proposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(proposal, authentication); + } + + List getExistingGroupMembers(@Nonnull final Urn groupUrn, final String actorUrnStr) { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + + final EntityRelationships relationships = + _graphClient.getRelatedEntities(groupUrn.toString(), ImmutableList.of(IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME), + RelationshipDirection.INCOMING, 0, 500, actorUrnStr); + return relationships.getRelationships().stream().map(EntityRelationship::getEntity).collect(Collectors.toList()); + } + + void removeExistingGroupMembers(@Nonnull final Urn groupUrn, @Nonnull final List userUrnList, + final Authentication authentication) throws Exception { + Objects.requireNonNull(groupUrn, "groupUrn must not be null"); + Objects.requireNonNull(userUrnList, "userUrnList must not be null"); + + final Set userUrns = new HashSet<>(userUrnList); + for (Urn userUrn : userUrns) { + final Map entityResponseMap = + _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, userUrns, Collections.singleton(GROUP_MEMBERSHIP_ASPECT_NAME), + authentication); + EntityResponse entityResponse = entityResponseMap.get(userUrn); + if (entityResponse == null) { + continue; + } + + final GroupMembership groupMembership = + new GroupMembership(entityResponse.getAspects().get(GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()); + if (groupMembership.getGroups().remove(groupUrn)) { + // Finally, create the MetadataChangeProposal. + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(userUrn); + proposal.setEntityType(CORP_USER_ENTITY_NAME); + proposal.setAspectName(GROUP_MEMBERSHIP_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(groupMembership)); + proposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(proposal, authentication); + } + } + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java index feadba96ec0b87..33d89e2dfad7d0 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java @@ -16,13 +16,6 @@ import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.ArrayUtils; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Date; import java.util.HashMap; @@ -31,11 +24,13 @@ import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang.ArrayUtils; -import static com.datahub.authentication.token.TokenClaims.ACTOR_ID_CLAIM_NAME; -import static com.datahub.authentication.token.TokenClaims.ACTOR_TYPE_CLAIM_NAME; -import static com.datahub.authentication.token.TokenClaims.TOKEN_TYPE_CLAIM_NAME; -import static com.datahub.authentication.token.TokenClaims.TOKEN_VERSION_CLAIM_NAME; +import static com.datahub.authentication.token.TokenClaims.*; /** @@ -48,7 +43,6 @@ public class StatefulTokenService extends StatelessTokenService { private final EntityService _entityService; private final LoadingCache _revokedTokenCache; private final String salt; - private final MessageDigest sha256; public StatefulTokenService(@Nonnull final String signingKey, @Nonnull final String signingAlgorithm, @Nullable final String iss, @Nonnull final EntityService entityService, @Nonnull final String salt) { @@ -56,7 +50,7 @@ public StatefulTokenService(@Nonnull final String signingKey, @Nonnull final Str this._entityService = entityService; this._revokedTokenCache = CacheBuilder.newBuilder() .maximumSize(10000) - .expireAfterWrite(6, TimeUnit.HOURS) + .expireAfterWrite(5, TimeUnit.MINUTES) .build(new CacheLoader() { @Override public Boolean load(final String key) { @@ -65,11 +59,6 @@ public Boolean load(final String key) { } }); this.salt = salt; - try { - this.sha256 = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Unable to get SHA-256 algorithm."); - } } /** @@ -161,7 +150,7 @@ public TokenClaims validateAccessToken(@Nonnull String accessToken) throws Token this.revokeAccessToken(hash(accessToken)); throw e; } catch (final ExecutionException e) { - throw new TokenException("Failed to validate DataHub token: Unable to load token information from store"); + throw new TokenException("Failed to validate DataHub token: Unable to load token information from store", e); } } @@ -174,7 +163,7 @@ public void revokeAccessToken(@Nonnull String hashedToken) throws TokenException return; } } catch (ExecutionException e) { - throw new TokenException("Failed to validate DataHub token from cache"); + throw new TokenException("Failed to validate DataHub token from cache", e); } throw new TokenException("Access token no longer exists"); } @@ -186,7 +175,7 @@ public String hash(String input) { final byte[] saltingKeyBytes = this.salt.getBytes(); final byte[] inputBytes = input.getBytes(); final byte[] concatBytes = ArrayUtils.addAll(inputBytes, saltingKeyBytes); - final byte[] bytes = sha256.digest(concatBytes); + final byte[] bytes = DigestUtils.sha256(concatBytes); return Base64.getEncoder().encodeToString(bytes); } } diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java new file mode 100644 index 00000000000000..007a22ba1da7fe --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/user/NativeUserService.java @@ -0,0 +1,295 @@ +package com.datahub.authentication.user; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.identity.CorpUserCredentials; +import com.linkedin.identity.CorpUserInfo; +import com.linkedin.identity.CorpUserStatus; +import com.linkedin.identity.InviteToken; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.secret.SecretService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.time.Instant; +import java.util.Base64; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.Constants.*; + + +/** + * Service responsible for creating, updating and authenticating native DataHub users. + */ +@Slf4j +public class NativeUserService { + private static final int LOWERCASE_ASCII_START = 97; + private static final int LOWERCASE_ASCII_END = 122; + private static final int INVITE_TOKEN_LENGTH = 32; + private static final int SALT_TOKEN_LENGTH = 16; + private static final int PASSWORD_RESET_TOKEN_LENGTH = 32; + private static final String HASHING_ALGORITHM = "SHA-256"; + private static final long ONE_DAY_MILLIS = TimeUnit.DAYS.toMillis(1); + + private final EntityService _entityService; + private final EntityClient _entityClient; + private final SecretService _secretService; + private final SecureRandom _secureRandom; + private final MessageDigest _messageDigest; + + public NativeUserService(@Nonnull EntityService entityService, @Nonnull EntityClient entityClient, @Nonnull SecretService secretService) + throws Exception { + Objects.requireNonNull(entityService, "entityService must not be null!"); + Objects.requireNonNull(entityClient, "entityClient must not be null!"); + Objects.requireNonNull(secretService, "secretService must not be null!"); + + _entityService = entityService; + _entityClient = entityClient; + _secretService = secretService; + _secureRandom = new SecureRandom(); + _messageDigest = MessageDigest.getInstance(HASHING_ALGORITHM); + } + + public void createNativeUser(@Nonnull String userUrnString, @Nonnull String fullName, @Nonnull String email, + @Nonnull String title, @Nonnull String password, @Nonnull String inviteToken, Authentication authentication) + throws Exception { + Objects.requireNonNull(userUrnString, "userUrnSting must not be null!"); + Objects.requireNonNull(fullName, "fullName must not be null!"); + Objects.requireNonNull(email, "email must not be null!"); + Objects.requireNonNull(title, "title must not be null!"); + Objects.requireNonNull(password, "password must not be null!"); + Objects.requireNonNull(inviteToken, "inviteToken must not be null!"); + + InviteToken inviteTokenAspect = + (InviteToken) _entityService.getLatestAspect(Urn.createFromString(GLOBAL_INVITE_TOKEN), + INVITE_TOKEN_ASPECT_NAME); + if (inviteTokenAspect == null || !inviteTokenAspect.hasToken() || !_secretService.decrypt( + inviteTokenAspect.getToken()).equals(inviteToken)) { + throw new RuntimeException("Invalid sign-up token. Please ask your administrator to send you an updated link!"); + } + + Urn userUrn = Urn.createFromString(userUrnString); + if (_entityService.exists(userUrn)) { + throw new RuntimeException("This user already exists! Cannot create a new user."); + } + updateCorpUserInfo(userUrn, fullName, email, title, authentication); + updateCorpUserStatus(userUrn, authentication); + updateCorpUserCredentials(userUrn, password, authentication); + } + + void updateCorpUserInfo(@Nonnull Urn userUrn, @Nonnull String fullName, @Nonnull String email, @Nonnull String title, + Authentication authentication) throws Exception { + // Construct corpUserInfo + final CorpUserInfo corpUserInfo = new CorpUserInfo(); + corpUserInfo.setFullName(fullName); + corpUserInfo.setDisplayName(fullName); + corpUserInfo.setEmail(email); + corpUserInfo.setTitle(title); + corpUserInfo.setActive(true); + + // Ingest corpUserInfo MCP + final MetadataChangeProposal corpUserInfoProposal = new MetadataChangeProposal(); + corpUserInfoProposal.setEntityType(CORP_USER_ENTITY_NAME); + corpUserInfoProposal.setEntityUrn(userUrn); + corpUserInfoProposal.setAspectName(CORP_USER_INFO_ASPECT_NAME); + corpUserInfoProposal.setAspect(GenericRecordUtils.serializeAspect(corpUserInfo)); + corpUserInfoProposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(corpUserInfoProposal, authentication); + } + + void updateCorpUserStatus(@Nonnull Urn userUrn, Authentication authentication) throws Exception { + // Construct corpUserStatus + CorpUserStatus corpUserStatus = new CorpUserStatus(); + corpUserStatus.setStatus(CORP_USER_STATUS_ACTIVE); + corpUserStatus.setLastModified( + new AuditStamp().setActor(Urn.createFromString(SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); + + // Ingest corpUserStatus MCP + final MetadataChangeProposal corpUserStatusProposal = new MetadataChangeProposal(); + corpUserStatusProposal.setEntityType(CORP_USER_ENTITY_NAME); + corpUserStatusProposal.setEntityUrn(userUrn); + corpUserStatusProposal.setAspectName(CORP_USER_STATUS_ASPECT_NAME); + corpUserStatusProposal.setAspect(GenericRecordUtils.serializeAspect(corpUserStatus)); + corpUserStatusProposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(corpUserStatusProposal, authentication); + } + + void updateCorpUserCredentials(@Nonnull Urn userUrn, @Nonnull String password, + Authentication authentication) throws Exception { + // Construct corpUserCredentials + CorpUserCredentials corpUserCredentials = new CorpUserCredentials(); + final byte[] salt = getRandomBytes(SALT_TOKEN_LENGTH); + String encryptedSalt = _secretService.encrypt(Base64.getEncoder().encodeToString(salt)); + corpUserCredentials.setSalt(encryptedSalt); + String hashedPassword = getHashedPassword(salt, password); + corpUserCredentials.setHashedPassword(hashedPassword); + + // Ingest corpUserCredentials MCP + final MetadataChangeProposal corpUserCredentialsProposal = new MetadataChangeProposal(); + corpUserCredentialsProposal.setEntityType(CORP_USER_ENTITY_NAME); + corpUserCredentialsProposal.setEntityUrn(userUrn); + corpUserCredentialsProposal.setAspectName(CORP_USER_CREDENTIALS_ASPECT_NAME); + corpUserCredentialsProposal.setAspect(GenericRecordUtils.serializeAspect(corpUserCredentials)); + corpUserCredentialsProposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(corpUserCredentialsProposal, authentication); + } + + public String generateNativeUserInviteToken(Authentication authentication) throws Exception { + // Construct inviteToken + InviteToken inviteToken = new InviteToken(); + String token = generateRandomLowercaseToken(INVITE_TOKEN_LENGTH); + inviteToken.setToken(_secretService.encrypt(token)); + + // Ingest corpUserCredentials MCP + final MetadataChangeProposal inviteTokenProposal = new MetadataChangeProposal(); + inviteTokenProposal.setEntityType(INVITE_TOKEN_ENTITY_NAME); + inviteTokenProposal.setEntityUrn(Urn.createFromString(GLOBAL_INVITE_TOKEN)); + inviteTokenProposal.setAspectName(INVITE_TOKEN_ASPECT_NAME); + inviteTokenProposal.setAspect(GenericRecordUtils.serializeAspect(inviteToken)); + inviteTokenProposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(inviteTokenProposal, authentication); + + return token; + } + + public String getNativeUserInviteToken(Authentication authentication) throws Exception { + InviteToken inviteToken = (InviteToken) _entityService.getLatestAspect(Urn.createFromString(GLOBAL_INVITE_TOKEN), + INVITE_TOKEN_ASPECT_NAME); + if (inviteToken == null || !inviteToken.hasToken()) { + return generateNativeUserInviteToken(authentication); + } + return _secretService.decrypt(inviteToken.getToken()); + } + + public String generateNativeUserPasswordResetToken(@Nonnull String userUrnString, + Authentication authentication) throws Exception { + Objects.requireNonNull(userUrnString, "userUrnString must not be null!"); + + Urn userUrn = Urn.createFromString(userUrnString); + + CorpUserCredentials corpUserCredentials = + (CorpUserCredentials) _entityService.getLatestAspect(userUrn, CORP_USER_CREDENTIALS_ASPECT_NAME); + if (corpUserCredentials == null || !corpUserCredentials.hasSalt() || !corpUserCredentials.hasHashedPassword()) { + throw new RuntimeException("User does not exist or is a non-native user!"); + } + // Add reset token to CorpUserCredentials + String passwordResetToken = generateRandomLowercaseToken(PASSWORD_RESET_TOKEN_LENGTH); + corpUserCredentials.setPasswordResetToken(_secretService.encrypt(passwordResetToken)); + + long expirationTime = Instant.now().plusMillis(ONE_DAY_MILLIS).toEpochMilli(); + corpUserCredentials.setPasswordResetTokenExpirationTimeMillis(expirationTime); + + // Ingest CorpUserCredentials MCP + final MetadataChangeProposal corpUserCredentialsProposal = new MetadataChangeProposal(); + corpUserCredentialsProposal.setEntityType(CORP_USER_ENTITY_NAME); + corpUserCredentialsProposal.setEntityUrn(userUrn); + corpUserCredentialsProposal.setAspectName(CORP_USER_CREDENTIALS_ASPECT_NAME); + corpUserCredentialsProposal.setAspect(GenericRecordUtils.serializeAspect(corpUserCredentials)); + corpUserCredentialsProposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(corpUserCredentialsProposal, authentication); + + return passwordResetToken; + } + + public void resetCorpUserCredentials(@Nonnull String userUrnString, @Nonnull String password, + @Nonnull String resetToken, Authentication authentication) throws Exception { + Objects.requireNonNull(userUrnString, "userUrnString must not be null!"); + Objects.requireNonNull(password, "password must not be null!"); + Objects.requireNonNull(resetToken, "resetToken must not be null!"); + + Urn userUrn = Urn.createFromString(userUrnString); + + CorpUserCredentials corpUserCredentials = + (CorpUserCredentials) _entityService.getLatestAspect(userUrn, CORP_USER_CREDENTIALS_ASPECT_NAME); + + if (corpUserCredentials == null || !corpUserCredentials.hasSalt() || !corpUserCredentials.hasHashedPassword()) { + throw new RuntimeException("User does not exist!"); + } + + if (!corpUserCredentials.hasPasswordResetToken() + || !corpUserCredentials.hasPasswordResetTokenExpirationTimeMillis() + || corpUserCredentials.getPasswordResetTokenExpirationTimeMillis() == null) { + throw new RuntimeException("User has not generated a password reset token!"); + } + + if (!_secretService.decrypt( + corpUserCredentials.getPasswordResetToken()).equals(resetToken)) { + throw new RuntimeException("Invalid reset token. Please ask your administrator to send you an updated link!"); + } + + long currentTimeMillis = Instant.now().toEpochMilli(); + if (currentTimeMillis > corpUserCredentials.getPasswordResetTokenExpirationTimeMillis()) { + throw new RuntimeException("Reset token has expired! Please ask your administrator to create a new one"); + } + + // Construct corpUserCredentials + final byte[] salt = getRandomBytes(SALT_TOKEN_LENGTH); + String encryptedSalt = _secretService.encrypt(Base64.getEncoder().encodeToString(salt)); + corpUserCredentials.setSalt(encryptedSalt); + String hashedPassword = getHashedPassword(salt, password); + corpUserCredentials.setHashedPassword(hashedPassword); + + // Ingest corpUserCredentials MCP + final MetadataChangeProposal corpUserCredentialsProposal = new MetadataChangeProposal(); + corpUserCredentialsProposal.setEntityType(CORP_USER_ENTITY_NAME); + corpUserCredentialsProposal.setEntityUrn(userUrn); + corpUserCredentialsProposal.setAspectName(CORP_USER_CREDENTIALS_ASPECT_NAME); + corpUserCredentialsProposal.setAspect(GenericRecordUtils.serializeAspect(corpUserCredentials)); + corpUserCredentialsProposal.setChangeType(ChangeType.UPSERT); + _entityClient.ingestProposal(corpUserCredentialsProposal, authentication); + } + + byte[] getRandomBytes(int length) { + byte[] randomBytes = new byte[length]; + _secureRandom.nextBytes(randomBytes); + return randomBytes; + } + + String generateRandomLowercaseToken(int length) { + return _secureRandom.ints(length, LOWERCASE_ASCII_START, LOWERCASE_ASCII_END + 1) + .mapToObj(i -> String.valueOf((char) i)) + .collect(Collectors.joining()); + } + + byte[] saltPassword(@Nonnull byte[] salt, @Nonnull String password) throws IOException { + byte[] passwordBytes = password.getBytes(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byteArrayOutputStream.write(salt); + byteArrayOutputStream.write(passwordBytes); + return byteArrayOutputStream.toByteArray(); + } + + public String getHashedPassword(@Nonnull byte[] salt, @Nonnull String password) throws IOException { + byte[] saltedPassword = saltPassword(salt, password); + byte[] hashedPassword = _messageDigest.digest(saltedPassword); + return Base64.getEncoder().encodeToString(hashedPassword); + } + + public boolean doesPasswordMatch(@Nonnull String userUrnString, @Nonnull String password) throws Exception { + Objects.requireNonNull(userUrnString, "userUrnSting must not be null!"); + Objects.requireNonNull(password, "Password must not be null!"); + + Urn userUrn = Urn.createFromString(userUrnString); + CorpUserCredentials corpUserCredentials = + (CorpUserCredentials) _entityService.getLatestAspect(userUrn, CORP_USER_CREDENTIALS_ASPECT_NAME); + if (corpUserCredentials == null || !corpUserCredentials.hasSalt() || !corpUserCredentials.hasHashedPassword()) { + return false; + } + + String decryptedSalt = _secretService.decrypt(corpUserCredentials.getSalt()); + byte[] salt = Base64.getDecoder().decode(decryptedSalt); + String storedHashedPassword = corpUserCredentials.getHashedPassword(); + String hashedPassword = getHashedPassword(salt, password); + return storedHashedPassword.equals(hashedPassword); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java index f66092bc529b8a..59f1e7dbb707e5 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizerChain.java @@ -25,8 +25,11 @@ public class AuthorizerChain implements Authorizer { private final List authorizers; - public AuthorizerChain(final List authorizers) { + private final Authorizer defaultAuthorizer; + + public AuthorizerChain(final List authorizers, Authorizer defaultAuthorizer) { this.authorizers = Objects.requireNonNull(authorizers); + this.defaultAuthorizer = defaultAuthorizer; } @Override @@ -116,13 +119,9 @@ private AuthorizedActors mergeAuthorizedActors(@Nullable AuthorizedActors origin } /** - * Returns an instance of {@link DataHubAuthorizer} if it is present in the Authentication chain, - * or null if it cannot be found. + * Returns an instance of default {@link DataHubAuthorizer} */ public DataHubAuthorizer getDefaultAuthorizer() { - return (DataHubAuthorizer) this.authorizers.stream() - .filter(authorizer -> authorizer instanceof DataHubAuthorizer) - .findFirst() - .orElse(null); + return (DataHubAuthorizer) defaultAuthorizer; } } \ No newline at end of file diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java index 4fa8a3efe8737f..3c146cc3f4d5b4 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java @@ -8,6 +8,7 @@ import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.identity.GroupMembership; +import com.linkedin.identity.NativeGroupMembership; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; @@ -30,9 +31,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.Constants.CORP_GROUP_ENTITY_NAME; -import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; -import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME; +import static com.linkedin.metadata.Constants.*; @Slf4j @@ -327,8 +326,12 @@ private Set resolveGroups(Urn actor, PolicyEvaluationContext context) { } Set groups = new HashSet<>(); - Optional maybeGroups = resolveGroupMembership(actor); - maybeGroups.ifPresent(groupMembership -> groups.addAll(groupMembership.getGroups())); + Optional maybeGroupMembership = resolveGroupMembership(actor); + maybeGroupMembership.ifPresent(groupMembership -> groups.addAll(groupMembership.getGroups())); + + Optional maybeNativeGroupMembership = resolveNativeGroupMembership(actor); + maybeNativeGroupMembership.ifPresent( + nativeGroupMembership -> groups.addAll(nativeGroupMembership.getNativeGroups())); context.setGroups(groups); // Cache the groups. return groups; } @@ -342,7 +345,22 @@ private Optional resolveGroupMembership(final Urn actor) { if (aspectMap.containsKey(GROUP_MEMBERSHIP_ASPECT_NAME)) { return Optional.of(new GroupMembership(aspectMap.get(GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data())); } + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException(String.format("Failed to fetch corpUser for urn %s", actor), e); + } + return Optional.empty(); + } + private Optional resolveNativeGroupMembership(final Urn actor) { + try { + final EntityResponse corpUser = + _entityClient.batchGetV2(CORP_USER_ENTITY_NAME, Collections.singleton(actor), null, _systemAuthentication) + .get(actor); + final EnvelopedAspectMap aspectMap = corpUser.getAspects(); + if (aspectMap.containsKey(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)) { + return Optional.of( + new NativeGroupMembership(aspectMap.get(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data())); + } } catch (RemoteInvocationException | URISyntaxException e) { throw new RuntimeException(String.format("Failed to fetch corpUser for urn %s", actor), e); } @@ -354,6 +372,7 @@ private Optional resolveGroupMembership(final Urn actor) { */ static class PolicyEvaluationContext { private Set groups; + public void setGroups(Set groups) { this.groups = groups; } diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticatorTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticatorTest.java index eaaf564ac4646f..c2b3e84b4a7c6c 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticatorTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/authenticator/DataHubTokenAuthenticatorTest.java @@ -6,6 +6,7 @@ import com.datahub.authentication.AuthenticationException; import com.datahub.authentication.AuthenticationRequest; import com.datahub.authentication.AuthenticatorContext; +import com.datahub.authentication.token.StatefulTokenService; import com.datahub.authentication.token.TokenType; import com.google.common.collect.ImmutableMap; import com.linkedin.common.urn.Urn; @@ -20,8 +21,7 @@ import java.util.Collections; import java.util.Map; -import static com.datahub.authentication.AuthenticationConstants.AUTHORIZATION_HEADER_NAME; -import static com.datahub.authentication.AuthenticationConstants.ENTITY_SERVICE; +import static com.datahub.authentication.AuthenticationConstants.*; import static com.datahub.authentication.authenticator.DataHubTokenAuthenticator.SALT_CONFIG_NAME; import static com.datahub.authentication.authenticator.DataHubTokenAuthenticator.SIGNING_ALG_CONFIG_NAME; import static com.datahub.authentication.authenticator.DataHubTokenAuthenticator.SIGNING_KEY_CONFIG_NAME; @@ -40,12 +40,13 @@ public class DataHubTokenAuthenticatorTest { private static final String TEST_SALT = "WnEdIeTG/VVCLQqGwC/BAkqyY0k+H8NEAtWGejrBI93="; final EntityService mockService = Mockito.mock(EntityService.class); + final StatefulTokenService statefulTokenService = new StatefulTokenService(TEST_SIGNING_KEY, "HS256", null, mockService, TEST_SALT); @Test public void testInit() { final DataHubTokenAuthenticator authenticator = new DataHubTokenAuthenticator(); AuthenticatorContext authenticatorContext = - new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService)); + new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService, TOKEN_SERVICE, statefulTokenService)); assertThrows(() -> authenticator.init(null, authenticatorContext)); assertThrows(() -> authenticator.init(Collections.emptyMap(), authenticatorContext)); assertThrows(() -> authenticator.init(ImmutableMap.of(SIGNING_KEY_CONFIG_NAME, TEST_SIGNING_KEY, @@ -64,7 +65,7 @@ public void testAuthenticateFailureMissingAuthorizationHeader() { authenticator.init(ImmutableMap.of(SIGNING_KEY_CONFIG_NAME, TEST_SIGNING_KEY, SALT_CONFIG_NAME, TEST_SALT, SIGNING_ALG_CONFIG_NAME, "HS256"), - new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService))); + new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService, TOKEN_SERVICE, statefulTokenService))); final AuthenticationRequest context = new AuthenticationRequest(Collections.emptyMap()); assertThrows(AuthenticationException.class, () -> authenticator.authenticate(context)); @@ -73,10 +74,9 @@ public void testAuthenticateFailureMissingAuthorizationHeader() { @Test public void testAuthenticateFailureMissingBearerCredentials() { final DataHubTokenAuthenticator authenticator = new DataHubTokenAuthenticator(); - authenticator.init(ImmutableMap.of(SIGNING_KEY_CONFIG_NAME, TEST_SIGNING_KEY, SALT_CONFIG_NAME, TEST_SALT, SIGNING_ALG_CONFIG_NAME, "HS256"), - new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService))); + new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService, TOKEN_SERVICE, statefulTokenService))); final AuthenticationRequest context = new AuthenticationRequest( ImmutableMap.of(AUTHORIZATION_HEADER_NAME, "Basic username:password") @@ -90,7 +90,7 @@ public void testAuthenticateFailureInvalidToken() { authenticator.init(ImmutableMap.of(SIGNING_KEY_CONFIG_NAME, TEST_SIGNING_KEY, SALT_CONFIG_NAME, TEST_SALT, SIGNING_ALG_CONFIG_NAME, "HS256"), - new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService))); + new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService, TOKEN_SERVICE, statefulTokenService))); final AuthenticationRequest context = new AuthenticationRequest( ImmutableMap.of(AUTHORIZATION_HEADER_NAME, "Bearer someRandomToken") @@ -111,7 +111,7 @@ public void testAuthenticateSuccess() throws Exception { final DataHubTokenAuthenticator authenticator = new DataHubTokenAuthenticator(); authenticator.init(ImmutableMap.of(SIGNING_KEY_CONFIG_NAME, TEST_SIGNING_KEY, SALT_CONFIG_NAME, TEST_SALT, SIGNING_ALG_CONFIG_NAME, "HS256"), - new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService))); + new AuthenticatorContext(ImmutableMap.of(ENTITY_SERVICE, mockService, TOKEN_SERVICE, statefulTokenService))); final Actor datahub = new Actor(ActorType.USER, "datahub"); final String validToken = authenticator._statefulTokenService.generateAccessToken( diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/group/GroupServiceTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/group/GroupServiceTest.java new file mode 100644 index 00000000000000..81cf94d3bfe029 --- /dev/null +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/group/GroupServiceTest.java @@ -0,0 +1,264 @@ +package com.datahub.authentication.group; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.EntityRelationship; +import com.linkedin.common.EntityRelationshipArray; +import com.linkedin.common.EntityRelationships; +import com.linkedin.common.Origin; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.identity.GroupMembership; +import com.linkedin.identity.NativeGroupMembership; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphClient; +import com.linkedin.metadata.key.CorpGroupKey; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class GroupServiceTest { + private static final String DATAHUB_SYSTEM_CLIENT_ID = "__datahub_system"; + + private static final String GROUP_NAME = "Group Name"; + private static final String GROUP_DESCRIPTION = "This is a group"; + private static final String GROUP_ID = "abcd"; + private static final String GROUP_URN_STRING = "urn:li:corpGroup:testNewGroup"; + private static final String NATIVE_GROUP_URN_STRING = "urn:li:corpGroup:testGroupNative"; + private static final String EXTERNAL_GROUP_URN_STRING = "urn:li:corpGroup:testGroupExternal"; + private static final String EMAIL = "mock@email.com"; + private static final Urn USER_URN = new CorpuserUrn(EMAIL); + private static final List USER_URN_LIST = new ArrayList<>(Collections.singleton(USER_URN)); + private static final Authentication SYSTEM_AUTHENTICATION = + new Authentication(new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID), ""); + + private static Urn _groupUrn; + private static CorpGroupKey _groupKey; + private static Map _entityResponseMap; + private static EntityRelationships _entityRelationships; + + private EntityClient _entityClient; + private EntityService _entityService; + private GraphClient _graphClient; + private GroupService _groupService; + + @BeforeMethod + public void setupTest() throws Exception { + _groupUrn = Urn.createFromString(GROUP_URN_STRING); + _groupKey = new CorpGroupKey(); + _groupKey.setName(GROUP_ID); + + NativeGroupMembership nativeGroupMembership = new NativeGroupMembership(); + nativeGroupMembership.setNativeGroups(new UrnArray(Urn.createFromString(NATIVE_GROUP_URN_STRING))); + GroupMembership groupMembership = new GroupMembership(); + groupMembership.setGroups(new UrnArray(Urn.createFromString(EXTERNAL_GROUP_URN_STRING))); + _entityResponseMap = ImmutableMap.of(USER_URN, new EntityResponse().setEntityName(CORP_USER_ENTITY_NAME) + .setUrn(USER_URN) + .setAspects(new EnvelopedAspectMap(ImmutableMap.of(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(nativeGroupMembership.data())), GROUP_MEMBERSHIP_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(groupMembership.data())))))); + + _entityRelationships = new EntityRelationships().setStart(0) + .setCount(1) + .setTotal(1) + .setRelationships(new EntityRelationshipArray(ImmutableList.of( + new EntityRelationship().setEntity(USER_URN).setType(IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME)))); + + _entityClient = mock(EntityClient.class); + _entityService = mock(EntityService.class); + _graphClient = mock(GraphClient.class); + + _groupService = new GroupService(_entityClient, _entityService, _graphClient); + } + + @Test + public void testConstructor() { + assertThrows(() -> new GroupService(null, _entityService, _graphClient)); + assertThrows(() -> new GroupService(_entityClient, null, _graphClient)); + assertThrows(() -> new GroupService(_entityClient, _entityService, null)); + + // Succeeds! + new GroupService(_entityClient, _entityService, _graphClient); + } + + @Test + public void testGroupExistsNullArguments() { + assertThrows(() -> _groupService.groupExists(null)); + } + + @Test + public void testGroupExistsPasses() { + when(_entityService.exists(_groupUrn)).thenReturn(true); + assertTrue(_groupService.groupExists(_groupUrn)); + } + + @Test + public void testGetGroupOriginNullArguments() { + assertThrows(() -> _groupService.getGroupOrigin(null)); + } + + @Test + public void testGetGroupOriginPasses() { + Origin groupOrigin = mock(Origin.class); + when(_entityService.getLatestAspect(eq(_groupUrn), eq(ORIGIN_ASPECT_NAME))).thenReturn(groupOrigin); + + assertEquals(groupOrigin, _groupService.getGroupOrigin(_groupUrn)); + } + + @Test + public void testAddUserToNativeGroupNullArguments() { + assertThrows(() -> _groupService.addUserToNativeGroup(null, _groupUrn, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.addUserToNativeGroup(USER_URN, null, SYSTEM_AUTHENTICATION)); + } + + @Test + public void testAddUserToNativeGroupPasses() throws Exception { + when(_entityService.exists(USER_URN)).thenReturn(true); + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), any(), any(), eq(SYSTEM_AUTHENTICATION))).thenReturn( + _entityResponseMap); + + _groupService.addUserToNativeGroup(USER_URN, _groupUrn, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testCreateNativeGroupNullArguments() { + assertThrows(() -> _groupService.createNativeGroup(null, GROUP_NAME, GROUP_DESCRIPTION, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.createNativeGroup(_groupKey, null, GROUP_DESCRIPTION, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.createNativeGroup(_groupKey, GROUP_NAME, null, SYSTEM_AUTHENTICATION)); + } + + @Test + public void testCreateNativeGroupPasses() throws Exception { + _groupService.createNativeGroup(_groupKey, GROUP_NAME, GROUP_DESCRIPTION, SYSTEM_AUTHENTICATION); + verify(_entityClient, times(2)).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testRemoveExistingNativeGroupMembersNullArguments() { + assertThrows(() -> _groupService.removeExistingNativeGroupMembers(null, USER_URN_LIST, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.removeExistingNativeGroupMembers(_groupUrn, null, SYSTEM_AUTHENTICATION)); + } + + @Test + public void testRemoveExistingNativeGroupMembersGroupNotInNativeGroupMembership() throws Exception { + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), any(), any(), eq(SYSTEM_AUTHENTICATION))).thenReturn( + _entityResponseMap); + + _groupService.removeExistingNativeGroupMembers(Urn.createFromString(EXTERNAL_GROUP_URN_STRING), USER_URN_LIST, + SYSTEM_AUTHENTICATION); + verify(_entityClient, never()).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testRemoveExistingNativeGroupMembersPasses() throws Exception { + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), any(), any(), eq(SYSTEM_AUTHENTICATION))).thenReturn( + _entityResponseMap); + + _groupService.removeExistingNativeGroupMembers(Urn.createFromString(NATIVE_GROUP_URN_STRING), USER_URN_LIST, + SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testMigrateGroupMembershipToNativeGroupMembershipNullArguments() { + assertThrows(() -> _groupService.migrateGroupMembershipToNativeGroupMembership(null, USER_URN.toString(), + SYSTEM_AUTHENTICATION)); + } + + @Test + public void testMigrateGroupMembershipToNativeGroupMembershipPasses() throws Exception { + when(_graphClient.getRelatedEntities(eq(EXTERNAL_GROUP_URN_STRING), + eq(ImmutableList.of(IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME)), eq(RelationshipDirection.INCOMING), anyInt(), + anyInt(), any())).thenReturn(_entityRelationships); + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), any(), any(), eq(SYSTEM_AUTHENTICATION))).thenReturn( + _entityResponseMap); + when(_entityService.exists(USER_URN)).thenReturn(true); + + _groupService.migrateGroupMembershipToNativeGroupMembership(Urn.createFromString(EXTERNAL_GROUP_URN_STRING), + USER_URN.toString(), SYSTEM_AUTHENTICATION); + verify(_entityClient, times(3)).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testCreateGroupInfoNullArguments() { + assertThrows(() -> _groupService.createGroupInfo(null, GROUP_NAME, GROUP_DESCRIPTION, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.createGroupInfo(_groupKey, null, GROUP_DESCRIPTION, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.createGroupInfo(_groupKey, GROUP_NAME, null, SYSTEM_AUTHENTICATION)); + } + + @Test + public void testCreateGroupInfoPasses() throws Exception { + _groupService.createGroupInfo(_groupKey, GROUP_NAME, GROUP_DESCRIPTION, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testCreateNativeGroupOriginNullArguments() { + assertThrows(() -> _groupService.createNativeGroupOrigin(null, SYSTEM_AUTHENTICATION)); + } + + @Test + public void testCreateNativeGroupOriginPasses() throws Exception { + _groupService.createNativeGroupOrigin(_groupUrn, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testGetExistingGroupMembersNullArguments() { + assertThrows(() -> _groupService.getExistingGroupMembers(null, USER_URN.toString())); + } + + @Test + public void testGetExistingGroupMembersPasses() { + when(_graphClient.getRelatedEntities(eq(GROUP_URN_STRING), + eq(ImmutableList.of(IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME)), eq(RelationshipDirection.INCOMING), anyInt(), + anyInt(), any())).thenReturn(_entityRelationships); + + assertEquals(USER_URN_LIST, _groupService.getExistingGroupMembers(_groupUrn, USER_URN.toString())); + } + + @Test + public void testRemoveExistingGroupMembersNullArguments() { + assertThrows(() -> _groupService.removeExistingGroupMembers(null, USER_URN_LIST, SYSTEM_AUTHENTICATION)); + assertThrows(() -> _groupService.removeExistingGroupMembers(_groupUrn, null, SYSTEM_AUTHENTICATION)); + } + + @Test + public void testRemoveExistingGroupMembersGroupNotInGroupMembership() throws Exception { + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), any(), any(), eq(SYSTEM_AUTHENTICATION))).thenReturn( + _entityResponseMap); + + _groupService.removeExistingGroupMembers(Urn.createFromString(NATIVE_GROUP_URN_STRING), USER_URN_LIST, + SYSTEM_AUTHENTICATION); + verify(_entityClient, never()).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } + + @Test + public void testRemoveExistingGroupMembersPasses() throws Exception { + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), any(), any(), eq(SYSTEM_AUTHENTICATION))).thenReturn( + _entityResponseMap); + + _groupService.removeExistingGroupMembers(Urn.createFromString(EXTERNAL_GROUP_URN_STRING), USER_URN_LIST, + SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), eq(SYSTEM_AUTHENTICATION)); + } +} diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java new file mode 100644 index 00000000000000..d19de607097c0a --- /dev/null +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authentication/user/NativeUserServiceTest.java @@ -0,0 +1,289 @@ +package com.datahub.authentication.user; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.identity.CorpUserCredentials; +import com.linkedin.identity.InviteToken; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.secret.SecretService; +import java.time.Instant; +import java.util.concurrent.TimeUnit; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; + + +public class NativeUserServiceTest { + private static final String DATAHUB_SYSTEM_CLIENT_ID = "__datahub_system"; + + private static final String USER_URN_STRING = "urn:li:corpuser:test"; + private static final String FULL_NAME = "MOCK NAME"; + private static final String EMAIL = "mock@email.com"; + private static final String TITLE = "Data Scientist"; + private static final String PASSWORD = "password"; + private static final String INVITE_TOKEN = "inviteToken"; + private static final String ENCRYPTED_INVITE_TOKEN = "encryptedInviteToken"; + private static final String RESET_TOKEN = "inviteToken"; + private static final String ENCRYPTED_RESET_TOKEN = "encryptedInviteToken"; + private static final String ENCRYPTED_SALT = "encryptedSalt"; + private static final Urn USER_URN = new CorpuserUrn(EMAIL); + private static final long ONE_DAY_MILLIS = TimeUnit.DAYS.toMillis(1); + private static final Authentication SYSTEM_AUTHENTICATION = + new Authentication(new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID), ""); + + private EntityService _entityService; + private EntityClient _entityClient; + private SecretService _secretService; + private NativeUserService _nativeUserService; + + @BeforeMethod + public void setupTest() throws Exception { + _entityService = mock(EntityService.class); + _entityClient = mock(EntityClient.class); + _secretService = mock(SecretService.class); + + _nativeUserService = new NativeUserService(_entityService, _entityClient, _secretService); + } + + @Test + public void testConstructor() throws Exception { + assertThrows(() -> new NativeUserService(null, _entityClient, _secretService)); + assertThrows(() -> new NativeUserService(_entityService, null, _secretService)); + assertThrows(() -> new NativeUserService(_entityService, _entityClient, null)); + + // Succeeds! + new NativeUserService(_entityService, _entityClient, _secretService); + } + + @Test + public void testCreateNativeUserNullArguments() { + assertThrows(() -> _nativeUserService.createNativeUser(null, FULL_NAME, EMAIL, TITLE, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION)); + assertThrows(() -> _nativeUserService.createNativeUser(USER_URN_STRING, null, EMAIL, TITLE, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION)); + assertThrows( + () -> _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, null, TITLE, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION)); + assertThrows( + () -> _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, null, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION)); + assertThrows(() -> _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, TITLE, null, INVITE_TOKEN, + SYSTEM_AUTHENTICATION)); + assertThrows(() -> _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, TITLE, PASSWORD, null, + SYSTEM_AUTHENTICATION)); + } + + @Test(expectedExceptions = RuntimeException.class, + expectedExceptionsMessageRegExp = "Invalid sign-up token. Please ask your administrator to send you an updated link!") + public void testCreateNativeUserInviteTokenDoesNotExist() throws Exception { + // Nonexistent invite token + when(_entityService.getLatestAspect(any(), eq(INVITE_TOKEN_ASPECT_NAME))).thenReturn(null); + + _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, TITLE, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION); + } + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "This user already exists! Cannot create a new user.") + public void testCreateNativeUserUserAlreadyExists() throws Exception { + InviteToken mockInviteTokenAspect = mock(InviteToken.class); + when(_entityService.getLatestAspect(any(), eq(INVITE_TOKEN_ASPECT_NAME))).thenReturn(mockInviteTokenAspect); + when(mockInviteTokenAspect.hasToken()).thenReturn(true); + when(mockInviteTokenAspect.getToken()).thenReturn(ENCRYPTED_INVITE_TOKEN); + when(_secretService.decrypt(eq(ENCRYPTED_INVITE_TOKEN))).thenReturn(INVITE_TOKEN); + + // The user already exists + when(_entityService.exists(any())).thenReturn(true); + + _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, TITLE, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION); + } + + @Test + public void testCreateNativeUserPasses() throws Exception { + InviteToken mockInviteTokenAspect = mock(InviteToken.class); + when(_entityService.getLatestAspect(any(), eq(INVITE_TOKEN_ASPECT_NAME))).thenReturn(mockInviteTokenAspect); + when(mockInviteTokenAspect.hasToken()).thenReturn(true); + when(mockInviteTokenAspect.getToken()).thenReturn(ENCRYPTED_INVITE_TOKEN); + when(_entityService.exists(any())).thenReturn(false); + when(_secretService.decrypt(eq(ENCRYPTED_INVITE_TOKEN))).thenReturn(INVITE_TOKEN); + when(_secretService.encrypt(any())).thenReturn(ENCRYPTED_SALT); + + _nativeUserService.createNativeUser(USER_URN_STRING, FULL_NAME, EMAIL, TITLE, PASSWORD, INVITE_TOKEN, + SYSTEM_AUTHENTICATION); + } + + @Test + public void testUpdateCorpUserInfoPasses() throws Exception { + _nativeUserService.updateCorpUserInfo(USER_URN, FULL_NAME, EMAIL, TITLE, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testUpdateCorpUserStatusPasses() throws Exception { + _nativeUserService.updateCorpUserStatus(USER_URN, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testUpdateCorpUserCredentialsPasses() throws Exception { + when(_secretService.encrypt(any())).thenReturn(ENCRYPTED_SALT); + + _nativeUserService.updateCorpUserCredentials(USER_URN, PASSWORD, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testGenerateNativeUserInviteTokenPasses() throws Exception { + when(_secretService.encrypt(any())).thenReturn(ENCRYPTED_INVITE_TOKEN); + + _nativeUserService.generateNativeUserInviteToken(SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testGetNativeUserInviteTokenInviteTokenDoesNotExistPasses() throws Exception { + // Nonexistent invite token + when(_entityService.getLatestAspect(any(), eq(INVITE_TOKEN_ASPECT_NAME))).thenReturn(null); + when(_secretService.encrypt(any())).thenReturn(ENCRYPTED_INVITE_TOKEN); + + _nativeUserService.getNativeUserInviteToken(SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testGetNativeUserInviteTokenPasses() throws Exception { + InviteToken mockInviteTokenAspect = mock(InviteToken.class); + when(_entityService.getLatestAspect(any(), eq(INVITE_TOKEN_ASPECT_NAME))).thenReturn(mockInviteTokenAspect); + when(_entityService.exists(any())).thenReturn(false); + when(mockInviteTokenAspect.hasToken()).thenReturn(true); + when(mockInviteTokenAspect.getToken()).thenReturn(ENCRYPTED_INVITE_TOKEN); + when(_secretService.decrypt(eq(ENCRYPTED_INVITE_TOKEN))).thenReturn(INVITE_TOKEN); + + assertEquals(_nativeUserService.getNativeUserInviteToken(SYSTEM_AUTHENTICATION), INVITE_TOKEN); + } + + @Test + public void testGenerateNativeUserResetTokenNullArguments() { + assertThrows(() -> _nativeUserService.generateNativeUserPasswordResetToken(null, SYSTEM_AUTHENTICATION)); + } + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "User does not exist or is a non-native user!") + public void testGenerateNativeUserResetTokenNotNativeUser() throws Exception { + // Nonexistent corpUserCredentials + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn(null); + + _nativeUserService.generateNativeUserPasswordResetToken(USER_URN_STRING, SYSTEM_AUTHENTICATION); + } + + @Test + public void testGenerateNativeUserResetToken() throws Exception { + CorpUserCredentials mockCorpUserCredentialsAspect = mock(CorpUserCredentials.class); + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn( + mockCorpUserCredentialsAspect); + when(mockCorpUserCredentialsAspect.hasSalt()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasHashedPassword()).thenReturn(true); + + when(_secretService.encrypt(any())).thenReturn(ENCRYPTED_INVITE_TOKEN); + + _nativeUserService.generateNativeUserPasswordResetToken(USER_URN_STRING, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testResetCorpUserCredentialsNullArguments() { + assertThrows(() -> _nativeUserService.resetCorpUserCredentials(null, PASSWORD, RESET_TOKEN, SYSTEM_AUTHENTICATION)); + assertThrows( + () -> _nativeUserService.resetCorpUserCredentials(USER_URN_STRING, null, RESET_TOKEN, SYSTEM_AUTHENTICATION)); + assertThrows( + () -> _nativeUserService.resetCorpUserCredentials(USER_URN_STRING, PASSWORD, null, SYSTEM_AUTHENTICATION)); + } + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "User has not generated a password reset token!") + public void testResetCorpUserCredentialsNoPasswordResetToken() throws Exception { + CorpUserCredentials mockCorpUserCredentialsAspect = mock(CorpUserCredentials.class); + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn( + mockCorpUserCredentialsAspect); + when(mockCorpUserCredentialsAspect.hasSalt()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasHashedPassword()).thenReturn(true); + // No password reset token + when(mockCorpUserCredentialsAspect.hasPasswordResetToken()).thenReturn(false); + + _nativeUserService.resetCorpUserCredentials(USER_URN_STRING, PASSWORD, RESET_TOKEN, SYSTEM_AUTHENTICATION); + } + + @Test(expectedExceptions = RuntimeException.class, + expectedExceptionsMessageRegExp = "Invalid reset token. Please ask your administrator to send you an updated link!") + public void testResetCorpUserCredentialsBadResetToken() throws Exception { + CorpUserCredentials mockCorpUserCredentialsAspect = mock(CorpUserCredentials.class); + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn( + mockCorpUserCredentialsAspect); + when(mockCorpUserCredentialsAspect.hasSalt()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasHashedPassword()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasPasswordResetToken()).thenReturn(true); + when(mockCorpUserCredentialsAspect.getPasswordResetToken()).thenReturn(ENCRYPTED_RESET_TOKEN); + when(mockCorpUserCredentialsAspect.hasPasswordResetTokenExpirationTimeMillis()).thenReturn(true); + when(mockCorpUserCredentialsAspect.getPasswordResetTokenExpirationTimeMillis()).thenReturn( + Instant.now().toEpochMilli()); + // Reset token won't match + when(_secretService.decrypt(eq(ENCRYPTED_RESET_TOKEN))).thenReturn("badResetToken"); + + _nativeUserService.resetCorpUserCredentials(USER_URN_STRING, PASSWORD, RESET_TOKEN, SYSTEM_AUTHENTICATION); + } + + @Test(expectedExceptions = RuntimeException.class, + expectedExceptionsMessageRegExp = "Reset token has expired! Please ask your administrator to create a new one") + public void testResetCorpUserCredentialsExpiredResetToken() throws Exception { + CorpUserCredentials mockCorpUserCredentialsAspect = mock(CorpUserCredentials.class); + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn( + mockCorpUserCredentialsAspect); + when(mockCorpUserCredentialsAspect.hasSalt()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasHashedPassword()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasPasswordResetToken()).thenReturn(true); + when(mockCorpUserCredentialsAspect.getPasswordResetToken()).thenReturn(ENCRYPTED_RESET_TOKEN); + when(mockCorpUserCredentialsAspect.hasPasswordResetTokenExpirationTimeMillis()).thenReturn(true); + // Reset token expiration time will be before the system time when we run resetCorpUserCredentials + when(mockCorpUserCredentialsAspect.getPasswordResetTokenExpirationTimeMillis()).thenReturn(0L); + when(_secretService.decrypt(eq(ENCRYPTED_RESET_TOKEN))).thenReturn(RESET_TOKEN); + + _nativeUserService.resetCorpUserCredentials(USER_URN_STRING, PASSWORD, RESET_TOKEN, SYSTEM_AUTHENTICATION); + } + + @Test + public void testResetCorpUserCredentialsPasses() throws Exception { + CorpUserCredentials mockCorpUserCredentialsAspect = mock(CorpUserCredentials.class); + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn( + mockCorpUserCredentialsAspect); + when(mockCorpUserCredentialsAspect.hasSalt()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasHashedPassword()).thenReturn(true); + when(mockCorpUserCredentialsAspect.hasPasswordResetToken()).thenReturn(true); + when(mockCorpUserCredentialsAspect.getPasswordResetToken()).thenReturn(ENCRYPTED_RESET_TOKEN); + when(mockCorpUserCredentialsAspect.hasPasswordResetTokenExpirationTimeMillis()).thenReturn(true); + when(mockCorpUserCredentialsAspect.getPasswordResetTokenExpirationTimeMillis()).thenReturn( + Instant.now().plusMillis(ONE_DAY_MILLIS).toEpochMilli()); + when(_secretService.decrypt(eq(ENCRYPTED_RESET_TOKEN))).thenReturn(RESET_TOKEN); + when(_secretService.encrypt(any())).thenReturn(ENCRYPTED_SALT); + + _nativeUserService.resetCorpUserCredentials(USER_URN_STRING, PASSWORD, RESET_TOKEN, SYSTEM_AUTHENTICATION); + verify(_entityClient).ingestProposal(any(), any()); + } + + @Test + public void testDoesPasswordMatchNullArguments() { + assertThrows(() -> _nativeUserService.doesPasswordMatch(null, PASSWORD)); + assertThrows(() -> _nativeUserService.doesPasswordMatch(USER_URN_STRING, null)); + } + + @Test + public void testDoesPasswordMatchNoCorpUserCredentials() throws Exception { + when(_entityService.getLatestAspect(any(), eq(CORP_USER_CREDENTIALS_ASPECT_NAME))).thenReturn(null); + + assertFalse(_nativeUserService.doesPasswordMatch(USER_URN_STRING, PASSWORD)); + } +} diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java index dba8c8924324c1..c5309734f42505 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java @@ -33,23 +33,10 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; -import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME; -import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME; -import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; -import static com.linkedin.metadata.authorization.PoliciesConfig.ACTIVE_POLICY_STATE; -import static com.linkedin.metadata.authorization.PoliciesConfig.INACTIVE_POLICY_STATE; -import static com.linkedin.metadata.authorization.PoliciesConfig.METADATA_POLICY_TYPE; -import static com.linkedin.metadata.authorization.PoliciesConfig.PLATFORM_POLICY_TYPE; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.authorization.PoliciesConfig.*; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; public class PolicyEngineTest { @@ -297,7 +284,7 @@ public void testEvaluatePolicyActorFilterGroupMatch() throws Exception { assertTrue(result1.isGranted()); // Verify we are only calling for group during these requests. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + verify(_entityClient, times(2)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); } @@ -334,7 +321,7 @@ public void testEvaluatePolicyActorFilterGroupNoMatch() throws Exception { assertFalse(result2.isGranted()); // Verify we are only calling for group during these requests. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), + verify(_entityClient, times(2)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), eq(null), any()); } @@ -413,9 +400,9 @@ public void testEvaluatePolicyActorFilterAllGroupsMatch() throws Exception { assertTrue(result2.isGranted()); // Verify we are only calling for group during these requests. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + verify(_entityClient, times(2)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), + verify(_entityClient, times(2)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), eq(null), any()); } @@ -484,8 +471,9 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersMatch() throws Excep Optional.of(resourceSpec)); assertTrue(result1.isGranted()); - // Ensure that caching of groups is working with 1 call to entity client for each principal. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + // Ensure that caching of groups is working with 2 calls (for groups and native groups) to entity client for each + // principal. + verify(_entityClient, times(2)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); } @@ -517,8 +505,9 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersNoMatch() throws Exc Optional.of(resourceSpec)); assertFalse(result2.isGranted()); - // Ensure that caching of groups is working with 1 call to entity client for each principal. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), + // Ensure that caching of groups is working with 2 calls (for groups and native groups) to entity client for each + // principal. + verify(_entityClient, times(2)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), eq(null), any()); } diff --git a/metadata-service/auth-ranger-impl/build.gradle b/metadata-service/auth-ranger-impl/build.gradle new file mode 100644 index 00000000000000..07ba87324d23c7 --- /dev/null +++ b/metadata-service/auth-ranger-impl/build.gradle @@ -0,0 +1,20 @@ +apply plugin: 'java' + + +compileJava { + options.debug = true +} + +dependencies { + compile project(path: ':metadata-service:auth-api') + + implementation 'org.apache.ranger:ranger-plugins-common:2.2.0' + implementation 'org.apache.logging.log4j:log4j-1.2-api:2.17.1' + + compile externalDependency.lombok + + annotationProcessor externalDependency.lombok + testCompile externalDependency.mockito + testCompile project(path: ':metadata-utils') + +} diff --git a/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/AuthorizerConfig.java b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/AuthorizerConfig.java new file mode 100644 index 00000000000000..3862df15dde2d6 --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/AuthorizerConfig.java @@ -0,0 +1,35 @@ +package com.datahub.authorization.ranger; + +import lombok.Builder; +import lombok.Getter; + + +@Getter +@Builder +public class AuthorizerConfig { + public static final String CONFIG_USERNAME = "username"; + public static final String CONFIG_PASSWORD = "password"; + + private final String username; + private final String password; + + public static CustomBuilder builder() { + return new CustomBuilder(); + } + + public static class CustomBuilder extends AuthorizerConfigBuilder { + + public AuthorizerConfig build() { + + if (super.username == null || super.username.trim().length() == 0) { + throw new IllegalArgumentException("username should empty"); + } + + if (super.password == null || super.password.trim().length() == 0) { + throw new IllegalArgumentException("password should empty"); + } + + return super.build(); + } + } +} diff --git a/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/DataHubRangerClient.java b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/DataHubRangerClient.java new file mode 100644 index 00000000000000..851cbf6e76ce7c --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/DataHubRangerClient.java @@ -0,0 +1,93 @@ +package com.datahub.authorization.ranger; + +import com.datahub.authorization.ranger.response.UserByName; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.util.StringUtils; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.service.RangerBasePlugin; + + +@Slf4j +public class DataHubRangerClient { + private static final String SERVICE_NAME = "datahub"; + private static final String PROP_RANGER_URL = StringUtils.format("ranger.plugin.%s.policy.rest.url", SERVICE_NAME); + private static final String PROP_RANGER_SSL = + StringUtils.format("ranger.plugin.%s.policy.rest.ssl.config.file", SERVICE_NAME); + // Apache Ranger base url + + private static RangerBasePlugin rangerBasePlugin; + private final AuthorizerConfig authorizerConfig; + private RangerRestClientWrapper rangerRestClientWrapper; + + public DataHubRangerClient(AuthorizerConfig authorizerConfig) { + this.authorizerConfig = authorizerConfig; + } + + public void init() { + + // No need of synchronization as single Authorizer object is getting created on server startup + if (rangerBasePlugin == null) { + // rangerBasePlugin is static + // Make sure classpath should have ranger-datahub-security.xml file + rangerBasePlugin = this.newRangerBasePlugin(); + rangerBasePlugin.setResultProcessor(new RangerDefaultAuditHandler()); + + rangerBasePlugin.init(); + log.info("Ranger policy engine is initialized"); + } + + rangerRestClientWrapper = this.newRangerRestClientWrapper(); + } + + public RangerBasePlugin newRangerBasePlugin() { + return new RangerBasePlugin(SERVICE_NAME, SERVICE_NAME); + } + + public RangerRestClientWrapper newRangerRestClientWrapper() { + String rangerURL = rangerBasePlugin.getConfig().get(PROP_RANGER_URL); + String rangerSslConfig = rangerBasePlugin.getConfig().get(PROP_RANGER_SSL, null); + RangerRestClientWrapper rangerRestClientWrapper = + new RangerRestClientWrapper(rangerURL, rangerSslConfig, this.authorizerConfig.getUsername(), + this.authorizerConfig.getPassword(), rangerBasePlugin.getConfig()); + return rangerRestClientWrapper; + } + + public Set getUserGroups(String userIdentifier) { + List groups = new ArrayList(); + + try { + + UserByName userByName = this.rangerRestClientWrapper.getUserByName(userIdentifier); + // userByName.id is (integer) apache ranger user identifier + groups = this.rangerRestClientWrapper.getUserById(userByName.getId()).getGroupNameList(); + + log.debug(StringUtils.format("User %s groups %s", userIdentifier, groups.toString())); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return groups.stream().collect(Collectors.toSet()); + } + + public Set getUserRoles(String userIdentifier) { + List roles = new ArrayList(); + try { + roles = this.rangerRestClientWrapper.getUserRole(userIdentifier); + log.debug(StringUtils.format("User %s roles %s", userIdentifier, roles.toString())); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return roles.stream().collect(Collectors.toSet()); + } + + public RangerAccessResult isAccessAllowed(RangerAccessRequest rangerAccessRequest) { + return rangerBasePlugin.isAccessAllowed(rangerAccessRequest); + } +} diff --git a/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/RangerAuthorizer.java b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/RangerAuthorizer.java new file mode 100644 index 00000000000000..178e94610ae8b3 --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/RangerAuthorizer.java @@ -0,0 +1,90 @@ +package com.datahub.authorization.ranger; + +import com.datahub.authorization.AuthorizationRequest; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.authorization.AuthorizedActors; +import com.datahub.authorization.Authorizer; +import com.datahub.authorization.AuthorizerContext; +import com.datahub.authorization.ResourceSpec; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import java.util.ArrayList; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; + + +@Slf4j +public class RangerAuthorizer implements Authorizer { + + private AuthorizerConfig authorizerConfig; + private DataHubRangerClient dataHubRangerClient; + + public RangerAuthorizer() { + } + + @Override + public void init(@Nonnull Map authorizerConfigMap, @Nonnull final AuthorizerContext ctx) { + this.authorizerConfig = AuthorizerConfig.builder() + .username((String) authorizerConfigMap.get(AuthorizerConfig.CONFIG_USERNAME)) + .password((String) authorizerConfigMap.get(AuthorizerConfig.CONFIG_PASSWORD)) + .build(); + + this.dataHubRangerClient = this.newDataHubRangerClient(); + this.dataHubRangerClient.init(); + } + + public DataHubRangerClient newDataHubRangerClient() { + return new DataHubRangerClient(this.authorizerConfig); + } + + @Override + public AuthorizationResult authorize(AuthorizationRequest request) { + + String userIdentifier = UrnUtils.getUrn(request.getActorUrn()).getId(); + + Set roles = this.dataHubRangerClient.getUserRoles(userIdentifier); + // getUserGroups is internally calling two API to get group information of Actor + Set groups = this.dataHubRangerClient.getUserGroups(userIdentifier); + + // set ResourceSpec default to "platform" + ResourceSpec resourceSpec = request.getResourceSpec().orElse(new ResourceSpec("platform", "platform")); + + // user has requested access to specific resource + log.debug(String.format("User \"%s\" requested access", userIdentifier)); + log.debug(String.format("Access is requested for resource type: %s", resourceSpec.getType())); + log.debug(String.format("Access is requested for resource : %s", resourceSpec.getResource())); + log.debug(String.format("Requested privilege : %s", request.getPrivilege())); + + // Convert resource type to lowercase as ranger doesn't support capital letter in resource type + RangerAccessResourceImpl rangerAccessResource = new RangerAccessResourceImpl(); + rangerAccessResource.setValue(resourceSpec.getType().toLowerCase(), resourceSpec.getResource()); + RangerAccessRequest rangerAccessRequest = + new RangerAccessRequestImpl(rangerAccessResource, request.getPrivilege(), userIdentifier, groups, roles); + + // Check with Apache Ranger if access is allowed to the user + RangerAccessResult accessResult = this.dataHubRangerClient.isAccessAllowed(rangerAccessRequest); + AuthorizationResult.Type result = AuthorizationResult.Type.DENY; + + if (accessResult != null && accessResult.getIsAllowed()) { + result = AuthorizationResult.Type.ALLOW; + } + + String message = String.format("Access to resource \"%s\" for privilege \"%s\" is \"%s\" for user \"%s\"", + resourceSpec.getResource(), request.getPrivilege(), result, userIdentifier); + log.debug(message); + return new AuthorizationResult(request, result, message); + } + + @Override + public AuthorizedActors authorizedActors(String privilege, Optional resourceSpec) { + log.info("Apache Ranger authorizer authorizedActors"); + return new AuthorizedActors(privilege, new ArrayList(), new ArrayList(), true, true); + } +} diff --git a/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/RangerRestClientWrapper.java b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/RangerRestClientWrapper.java new file mode 100644 index 00000000000000..03b4f234046e1e --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/RangerRestClientWrapper.java @@ -0,0 +1,52 @@ +package com.datahub.authorization.ranger; + +import com.datahub.authorization.ranger.response.UserById; +import com.datahub.authorization.ranger.response.UserByName; +import com.sun.jersey.api.client.ClientResponse; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.util.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.util.RangerRESTClient; + + +/** + * Class to wrap Apache Ranger Rest Client + */ + +public class RangerRestClientWrapper { + private static final String URI_BASE = "/service"; + private static final String USER_ROLES = URI_BASE + "/roles/roles/user/%s"; + private static final String USER_BY_NAME = URI_BASE + "/xusers/users/userName/%s"; + private static final String USER_BY_ID = URI_BASE + "/xusers/users/%d"; + + private final RangerRESTClient rangerRESTClient; + + public RangerRestClientWrapper(String rangerUrl, String rangerSslConfig, String userName, String password, + RangerPluginConfig pluginConfig) { + this.rangerRESTClient = new RangerRESTClient(rangerUrl, rangerSslConfig, pluginConfig); + this.rangerRESTClient.setBasicAuthInfo(userName, password); + } + + public UserByName getUserByName(String userName) throws Exception { + ClientResponse clientResponse = this.rangerRESTClient.get(StringUtils.format(USER_BY_NAME, userName), null); + Map userByNameMap = clientResponse.getEntity(new HashMap().getClass()); + UserByName userByNameResponse = new UserByName(userByNameMap); + return userByNameResponse; + } + + public UserById getUserById(Integer id) throws Exception { + ClientResponse clientResponse = this.rangerRESTClient.get(StringUtils.format(USER_BY_ID, id), null); + Map userByIdMap = clientResponse.getEntity(new HashMap().getClass()); + UserById userByIdResponse = new UserById(userByIdMap); + return userByIdResponse; + } + + public List getUserRole(String username) throws Exception { + ClientResponse clientResponse = + this.rangerRESTClient.get(StringUtils.format(USER_ROLES, username), null); + return clientResponse.getEntity((new ArrayList()).getClass()); + } +} \ No newline at end of file diff --git a/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/response/UserById.java b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/response/UserById.java new file mode 100644 index 00000000000000..cd4f91a0aa837a --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/response/UserById.java @@ -0,0 +1,24 @@ +package com.datahub.authorization.ranger.response; + +import java.util.List; +import java.util.Map; + + +public class UserById { + public static final String GROUP_NAME_LIST = "groupNameList"; + private final List groupNameList; + + public UserById(Map userPropertyMap) throws Exception { + if (!userPropertyMap.containsKey(GROUP_NAME_LIST)) { + throw new Exception(String.format("Property \"%s\" is not found", GROUP_NAME_LIST)); + } + + this.groupNameList = (List) userPropertyMap.get(GROUP_NAME_LIST); + } + + public List getGroupNameList() { + return this.groupNameList; + } +} + + diff --git a/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/response/UserByName.java b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/response/UserByName.java new file mode 100644 index 00000000000000..3136d7a7d0ca8d --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/main/java/com/datahub/authorization/ranger/response/UserByName.java @@ -0,0 +1,22 @@ +package com.datahub.authorization.ranger.response; + +import java.util.Map; +import org.apache.hadoop.util.StringUtils; + + +public class UserByName { + public static final String ID = "id"; + private final Integer id; + + public UserByName(Map userPropertyMap) throws Exception { + if (!userPropertyMap.containsKey(ID)) { + throw new Exception(StringUtils.format("Property \"%s\" is not found", ID)); + } + + this.id = (Integer) userPropertyMap.get(ID); + } + + public Integer getId() { + return this.id; + } +} diff --git a/metadata-service/auth-ranger-impl/src/test/java/com/datahub/authorization/ranger/RangerAuthorizerTest.java b/metadata-service/auth-ranger-impl/src/test/java/com/datahub/authorization/ranger/RangerAuthorizerTest.java new file mode 100644 index 00000000000000..cefca552e7d134 --- /dev/null +++ b/metadata-service/auth-ranger-impl/src/test/java/com/datahub/authorization/ranger/RangerAuthorizerTest.java @@ -0,0 +1,150 @@ +package com.datahub.authorization.ranger; + +import com.datahub.authorization.AuthorizationRequest; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.authorization.AuthorizerContext; +import com.datahub.authorization.ranger.response.UserById; +import com.datahub.authorization.ranger.response.UserByName; +import com.linkedin.metadata.authorization.PoliciesConfig; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.mockito.ArgumentMatcher; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static org.mockito.Mockito.*; + + +/*** + * RangerAccessRequest class doesn't have the equal method implementation and hence need to provide mock matcher + */ +class RangerAccessRequestMatcher implements ArgumentMatcher { + private final RangerAccessRequest expected; + + public RangerAccessRequestMatcher(RangerAccessRequest expected) { + this.expected = expected; + } + + @Override + public boolean matches(RangerAccessRequest argument) { + return argument.getUserGroups().equals(expected.getUserGroups()) && argument.getUserRoles() + .equals(expected.getUserRoles()) && argument.getUser().equals(expected.getUser()); + } +} + +public class RangerAuthorizerTest { + private RangerBasePlugin rangerBasePlugin; + private RangerRestClientWrapper rangerRestClientWrapper; + + private DataHubRangerClient _dataHubRangerClient; + + private RangerAuthorizer rangerAuthorizer; + private List roles; + private List groups; + + RangerAccessResourceImpl rangerAccessResource; + private Map authorizerConfigMap; + + @BeforeMethod + public void setupTest() throws Exception { + authorizerConfigMap = new HashMap<>(); + authorizerConfigMap.put(AuthorizerConfig.CONFIG_USERNAME, "foo"); + authorizerConfigMap.put(AuthorizerConfig.CONFIG_PASSWORD, "bar"); + + // Mock Apache Ranger library classes + rangerBasePlugin = mock(RangerBasePlugin.class); + rangerRestClientWrapper = mock(RangerRestClientWrapper.class); + + // Spy our class method to inject Mock objects of Apache Ranger library classes + rangerAuthorizer = spy(RangerAuthorizer.class); + + AuthorizerConfig authorizerConfig = AuthorizerConfig.builder() + .username((String) authorizerConfigMap.get(AuthorizerConfig.CONFIG_USERNAME)) + .password((String) authorizerConfigMap.get(AuthorizerConfig.CONFIG_PASSWORD)) + .build(); + + _dataHubRangerClient = spy(new DataHubRangerClient(authorizerConfig)); + + rangerAccessResource = new RangerAccessResourceImpl(); + rangerAccessResource.setValue("platform", "platform"); + + // Mock + doNothing().when(rangerBasePlugin).setResultProcessor(null); + doNothing().when(rangerBasePlugin).init(); + doReturn(rangerBasePlugin).when(_dataHubRangerClient).newRangerBasePlugin(); + doReturn(rangerRestClientWrapper).when(_dataHubRangerClient).newRangerRestClientWrapper(); + doReturn(_dataHubRangerClient).when(rangerAuthorizer).newDataHubRangerClient(); + + roles = new ArrayList<>(); + roles.add("admin"); + when(rangerRestClientWrapper.getUserRole("datahub")).thenReturn(roles); + + Map userByIdResponse = new HashMap<>(); + groups = new ArrayList<>(); + groups.add("public"); + userByIdResponse.put(UserById.GROUP_NAME_LIST, groups); + when(rangerRestClientWrapper.getUserById(1)).thenReturn(new UserById(userByIdResponse)); + + when(_dataHubRangerClient.newRangerBasePlugin()).thenReturn(rangerBasePlugin); + + rangerAuthorizer.init(authorizerConfigMap, new AuthorizerContext(null)); + } + + @Test + public void testAuthorizationAllow() throws Exception { + + RangerAccessRequest rangerAccessRequest = + new RangerAccessRequestImpl(rangerAccessResource, PoliciesConfig.VIEW_ANALYTICS_PRIVILEGE.getType(), "datahub", + this.groups.stream().collect(Collectors.toSet()), this.roles.stream().collect(Collectors.toSet())); + + RangerAccessResult rangerAccessResult = new RangerAccessResult(1, "datahub", null, null); + // For rangerAccessRequest the access should be allowed + rangerAccessResult.setIsAllowed(true); + + when(rangerBasePlugin.isAccessAllowed(argThat(new RangerAccessRequestMatcher(rangerAccessRequest)))).thenReturn( + rangerAccessResult); + // mock Apache Ranger API response as per username "github" + Map userByNameResponse = new HashMap<>(); + userByNameResponse.put(UserByName.ID, 1); + when(rangerRestClientWrapper.getUserByName("datahub")).thenReturn(new UserByName(userByNameResponse)); + + assert this.callAuthorizer("urn:li:corpuser:datahub").getType() == AuthorizationResult.Type.ALLOW; + } + + @Test + public void testAuthorizationDeny() throws Exception { + + RangerAccessRequest rangerAccessRequest = + new RangerAccessRequestImpl(rangerAccessResource, PoliciesConfig.VIEW_ANALYTICS_PRIVILEGE.getType(), "X", + this.groups.stream().collect(Collectors.toSet()), this.roles.stream().collect(Collectors.toSet())); + + RangerAccessResult rangerAccessResult = new RangerAccessResult(1, "datahub", null, null); + // For rangerAccessRequest the access should be denied + rangerAccessResult.setIsAllowed(false); + + // mock Apache Ranger API response as per username "X" + Map userByNameResponse = new HashMap<>(); + userByNameResponse.put(UserByName.ID, 1); + when(rangerRestClientWrapper.getUserByName("X")).thenReturn(new UserByName(userByNameResponse)); + + when(rangerBasePlugin.isAccessAllowed(argThat(new RangerAccessRequestMatcher(rangerAccessRequest)))).thenReturn( + rangerAccessResult); + + assert this.callAuthorizer("urn:li:corpuser:X").getType() == AuthorizationResult.Type.DENY; + } + + private AuthorizationResult callAuthorizer(String urn) { + AuthorizationRequest authorizationRequest = + new AuthorizationRequest(urn, PoliciesConfig.VIEW_ANALYTICS_PRIVILEGE.getType(), Optional.empty()); + return rangerAuthorizer.authorize(authorizationRequest); + } +} \ No newline at end of file diff --git a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/authentication/AuthServiceController.java b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/authentication/AuthServiceController.java index 4b6f0a31abb9b7..60de2de9a8689d 100644 --- a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/authentication/AuthServiceController.java +++ b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/authentication/AuthServiceController.java @@ -1,7 +1,8 @@ package com.datahub.authentication; -import com.datahub.authentication.token.TokenType; import com.datahub.authentication.token.StatelessTokenService; +import com.datahub.authentication.token.TokenType; +import com.datahub.authentication.user.NativeUserService; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,6 +26,16 @@ public class AuthServiceController { private static final String USER_ID_FIELD_NAME = "userId"; private static final String ACCESS_TOKEN_FIELD_NAME = "accessToken"; + private static final String USER_URN_FIELD_NAME = "userUrn"; + private static final String FULL_NAME_FIELD_NAME = "fullName"; + private static final String EMAIL_FIELD_NAME = "email"; + private static final String TITLE_FIELD_NAME = "title"; + private static final String PASSWORD_FIELD_NAME = "password"; + private static final String INVITE_TOKEN_FIELD_NAME = "inviteToken"; + private static final String RESET_TOKEN_FIELD_NAME = "resetToken"; + private static final String IS_NATIVE_USER_CREATED_FIELD_NAME = "isNativeUserCreated"; + private static final String ARE_NATIVE_USER_CREDENTIALS_RESET_FIELD_NAME = "areNativeUserCredentialsReset"; + private static final String DOES_PASSWORD_MATCH_FIELD_NAME = "doesPasswordMatch"; @Inject StatelessTokenService _statelessTokenService; @@ -35,13 +46,16 @@ public class AuthServiceController { @Inject ConfigurationProvider _configProvider; + @Inject + NativeUserService _nativeUserService; + /** * Generates a JWT access token for as user UI session, provided a unique "user id" to generate the token for inside a JSON * POST body. * * Example Request: * - * POST /generateSessionToken -H "Authorization: Basic :" + * POST /generateSessionTokenForUser -H "Authorization: Basic :" * { * "userId": "datahub" * } @@ -95,7 +109,192 @@ CompletableFuture> generateSessionTokenForUser(final Http }); } - // Currently only internal system is authorized to generate a token on behalf of a user! + /** + * Creates a native DataHub user using the provided full name, email and password. The provided invite token must + * be current otherwise a new user will not be created. + * + * Example Request: + * + * POST /signUp -H "Authorization: Basic :" + * { + * "fullName": "Full Name" + * "userUrn": "urn:li:corpuser:test" + * "email": "email@test.com" + * "title": "Data Scientist" + * "password": "password123" + * "inviteToken": "abcd" + * } + * + * Example Response: + * + * { + * "isNativeUserCreated": true + * } + */ + @PostMapping(value = "/signUp", produces = "application/json;charset=utf-8") + CompletableFuture> signUp(final HttpEntity httpEntity) { + String jsonStr = httpEntity.getBody(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; + try { + bodyJson = mapper.readTree(jsonStr); + } catch (JsonProcessingException e) { + log.error(String.format("Failed to parse json while attempting to create native user %s", jsonStr)); + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + if (bodyJson == null) { + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + /* + * Extract username and password field + */ + JsonNode userUrn = bodyJson.get(USER_URN_FIELD_NAME); + JsonNode fullName = bodyJson.get(FULL_NAME_FIELD_NAME); + JsonNode email = bodyJson.get(EMAIL_FIELD_NAME); + JsonNode title = bodyJson.get(TITLE_FIELD_NAME); + JsonNode password = bodyJson.get(PASSWORD_FIELD_NAME); + JsonNode inviteToken = bodyJson.get(INVITE_TOKEN_FIELD_NAME); + if (fullName == null || userUrn == null || email == null || title == null || password == null + || inviteToken == null) { + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + + String userUrnString = userUrn.asText(); + String fullNameString = fullName.asText(); + String emailString = email.asText(); + String titleString = title.asText(); + String passwordString = password.asText(); + String inviteTokenString = inviteToken.asText(); + log.debug(String.format("Attempting to create credentials for native user %s", userUrnString)); + return CompletableFuture.supplyAsync(() -> { + try { + _nativeUserService.createNativeUser(userUrnString, fullNameString, emailString, titleString, passwordString, + inviteTokenString, AuthenticationContext.getAuthentication()); + String response = buildSignUpResponse(); + return new ResponseEntity<>(response, HttpStatus.OK); + } catch (Exception e) { + log.error(String.format("Failed to create credentials for native user %s", userUrnString), e); + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + }); + } + + /** + * Resets the credentials for a native DataHub user using the provided email and new password. The provided reset + * token must be current otherwise the credentials will not be updated + * + * Example Request: + * + * POST /resetNativeUserCredentials -H "Authorization: Basic :" + * { + * "userUrn": "urn:li:corpuser:test" + * "password": "password123" + * "resetToken": "abcd" + * } + * + * Example Response: + * + * { + * "areNativeUserCredentialsReset": true + * } + */ + @PostMapping(value = "/resetNativeUserCredentials", produces = "application/json;charset=utf-8") + CompletableFuture> resetNativeUserCredentials(final HttpEntity httpEntity) { + String jsonStr = httpEntity.getBody(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; + try { + bodyJson = mapper.readTree(jsonStr); + } catch (JsonProcessingException e) { + log.error(String.format("Failed to parse json while attempting to create native user %s", jsonStr)); + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + if (bodyJson == null) { + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + /* + * Extract username and password field + */ + JsonNode userUrn = bodyJson.get(USER_URN_FIELD_NAME); + JsonNode password = bodyJson.get(PASSWORD_FIELD_NAME); + JsonNode resetToken = bodyJson.get(RESET_TOKEN_FIELD_NAME); + if (userUrn == null || password == null || resetToken == null) { + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + + String userUrnString = userUrn.asText(); + String passwordString = password.asText(); + String resetTokenString = resetToken.asText(); + log.debug(String.format("Attempting to reset credentials for native user %s", userUrnString)); + return CompletableFuture.supplyAsync(() -> { + try { + _nativeUserService.resetCorpUserCredentials(userUrnString, passwordString, resetTokenString, + AuthenticationContext.getAuthentication()); + String response = buildResetNativeUserCredentialsResponse(); + return new ResponseEntity<>(response, HttpStatus.OK); + } catch (Exception e) { + log.error(String.format("Failed to reset credentials for native user %s", userUrnString), e); + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + }); + } + + /** + * Verifies the credentials for a native DataHub user. + * + * Example Request: + * + * POST /verifyNativeUserCredentials -H "Authorization: Basic :" + * { + * "userUrn": "urn:li:corpuser:test" + * "password": "password123" + * } + * + * Example Response: + * + * { + * "passwordMatches": true + * } + */ + @PostMapping(value = "/verifyNativeUserCredentials", produces = "application/json;charset=utf-8") + CompletableFuture> verifyNativeUserCredentials(final HttpEntity httpEntity) { + String jsonStr = httpEntity.getBody(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode bodyJson; + try { + bodyJson = mapper.readTree(jsonStr); + } catch (JsonProcessingException e) { + log.error(String.format("Failed to parse json while attempting to verify native user password %s", jsonStr)); + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + if (bodyJson == null) { + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + /* + * Extract username and password field + */ + JsonNode userUrn = bodyJson.get(USER_URN_FIELD_NAME); + JsonNode password = bodyJson.get(PASSWORD_FIELD_NAME); + if (userUrn == null || password == null) { + return CompletableFuture.completedFuture(new ResponseEntity<>(HttpStatus.BAD_REQUEST)); + } + + String userUrnString = userUrn.asText(); + String passwordString = password.asText(); + log.debug(String.format("Attempting to verify credentials for native user %s", userUrnString)); + return CompletableFuture.supplyAsync(() -> { + try { + boolean doesPasswordMatch = _nativeUserService.doesPasswordMatch(userUrnString, passwordString); + String response = buildVerifyNativeUserPasswordResponse(doesPasswordMatch); + return new ResponseEntity<>(response, HttpStatus.OK); + } catch (Exception e) { + log.error(String.format("Failed to verify credentials for native user %s", userUrnString), e); + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + }); + } + + // Currently, only internal system is authorized to generate a token on behalf of a user! private boolean isAuthorizedToGenerateSessionToken(final String actorId) { // Verify that the actor is an internal system caller. final String systemClientId = _systemAuthentication.getActor().getId(); @@ -107,4 +306,22 @@ private String buildTokenResponse(final String token) { json.put(ACCESS_TOKEN_FIELD_NAME, token); return json.toString(); } + + private String buildSignUpResponse() { + JSONObject json = new JSONObject(); + json.put(IS_NATIVE_USER_CREATED_FIELD_NAME, true); + return json.toString(); + } + + private String buildResetNativeUserCredentialsResponse() { + JSONObject json = new JSONObject(); + json.put(ARE_NATIVE_USER_CREDENTIALS_RESET_FIELD_NAME, true); + return json.toString(); + } + + private String buildVerifyNativeUserPasswordResponse(final boolean doesPasswordMatch) { + JSONObject json = new JSONObject(); + json.put(DOES_PASSWORD_MATCH_FIELD_NAME, doesPasswordMatch); + return json.toString(); + } } diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index 410424df92d74d..453e8727c48bbf 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -23,6 +23,7 @@ dependencies { compile externalDependency.springCore compile externalDependency.springKafka compile externalDependency.springWeb + compile project(':metadata-service:auth-ranger-impl') annotationProcessor externalDependency.lombok @@ -30,4 +31,5 @@ dependencies { testCompile externalDependency.mockito testCompile externalDependency.testng + } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java index fb1c21965d43d5..f9be50fc368fd4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java @@ -25,12 +25,12 @@ import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; + @Slf4j @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) @Import({DataHubAuthorizerFactory.class}) public class AuthorizerChainFactory { - @Autowired private ConfigurationProvider configurationProvider; @@ -52,12 +52,17 @@ public class AuthorizerChainFactory { protected AuthorizerChain getInstance() { // Init authorizer context final AuthorizerContext ctx = initAuthorizerContext(); + // Extract + initialize customer authorizers from application configs. final List authorizers = new ArrayList<>(initCustomAuthorizers(ctx)); - // Add the DataHub core policies-based Authorizer - this one should always be enabled. - this.dataHubAuthorizer.init(Collections.emptyMap(), ctx); - authorizers.add(this.dataHubAuthorizer); - return new AuthorizerChain(authorizers); + + if (configurationProvider.getAuthorization().getDefaultAuthorizer().isEnabled()) { + this.dataHubAuthorizer.init(Collections.emptyMap(), ctx); + log.info("Default DataHubAuthorizer is enabled. Appending it to the authorization chain."); + authorizers.add(this.dataHubAuthorizer); + } + + return new AuthorizerChain(authorizers, dataHubAuthorizer); } private AuthorizerContext initAuthorizerContext() { @@ -75,6 +80,12 @@ private List initCustomAuthorizers(AuthorizerContext ctx) { for (AuthorizerConfiguration authorizer : authorizerConfigurations) { final String type = authorizer.getType(); + // continue if authorizer is not enabled + if (!authorizer.isEnabled()) { + log.info(String.format("Authorizer %s is not enabled", type)); + continue; + } + final Map configs = authorizer.getConfigs() != null ? authorizer.getConfigs() : Collections.emptyMap(); @@ -94,8 +105,10 @@ private List initCustomAuthorizers(AuthorizerContext ctx) { final Authorizer authorizerInstance = clazz.newInstance(); authorizerInstance.init(configs, ctx); customAuthorizers.add(authorizerInstance); + log.info(String.format("Authorizer %s is initialized", type)); } catch (Exception e) { - throw new RuntimeException(String.format("Failed to instantiate custom Authorizer with class name %s", clazz.getCanonicalName()), e); + throw new RuntimeException( + String.format("Failed to instantiate custom Authorizer with class name %s", clazz.getCanonicalName()), e); } } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java index 997709b2c90f52..e5e377b5777c1e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java @@ -32,7 +32,7 @@ public class DataHubAuthorizerFactory { @Value("${authorization.defaultAuthorizer.cacheRefreshIntervalSecs}") private Integer policyCacheRefreshIntervalSeconds; - @Value("${authorization.defaultAuthorizer..enabled:true}") + @Value("${authorization.defaultAuthorizer.enabled:true}") private Boolean policiesEnabled; @Bean(name = "dataHubAuthorizer") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java new file mode 100644 index 00000000000000..fd77cb606c5442 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java @@ -0,0 +1,40 @@ + + +package com.linkedin.gms.factory.auth; + +import com.datahub.authentication.group.GroupService; +import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphClient; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.Scope; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class GroupServiceFactory { + @Autowired + @Qualifier("entityService") + private EntityService _entityService; + + @Autowired + @Qualifier("javaEntityClient") + private JavaEntityClient _javaEntityClient; + + @Autowired + @Qualifier("graphClient") + private GraphClient _graphClient; + + @Bean(name = "groupService") + @Scope("singleton") + @Nonnull + protected GroupService getInstance() throws Exception { + return new GroupService(this._javaEntityClient, this._entityService, this._graphClient); + } +} \ No newline at end of file diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java new file mode 100644 index 00000000000000..9f510444a51aed --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java @@ -0,0 +1,40 @@ + + +package com.linkedin.gms.factory.auth; + +import com.datahub.authentication.user.NativeUserService; +import com.linkedin.entity.client.JavaEntityClient; +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.secret.SecretService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.Scope; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class NativeUserServiceFactory { + @Autowired + @Qualifier("entityService") + private EntityService _entityService; + + @Autowired + @Qualifier("javaEntityClient") + private JavaEntityClient _javaEntityClient; + + @Autowired + @Qualifier("dataHubSecretService") + private SecretService _secretService; + + @Bean(name = "nativeUserService") + @Scope("singleton") + @Nonnull + protected NativeUserService getInstance() throws Exception { + return new NativeUserService(this._entityService, this._javaEntityClient, this._secretService); + } +} \ No newline at end of file diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java new file mode 100644 index 00000000000000..3ba6965577204a --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java @@ -0,0 +1,34 @@ +package com.linkedin.gms.factory.common; + +import com.linkedin.gms.factory.entity.EntityServiceFactory; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.graph.SiblingGraphService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; + + +@Configuration +@Import({GraphServiceFactory.class, EntityServiceFactory.class}) +public class SiblingGraphServiceFactory { + + @Autowired + @Qualifier("entityService") + private EntityService _entityService; + + @Autowired + @Qualifier("graphService") + private GraphService graphService; + + @Bean(name = "siblingGraphService") + @Primary + @Nonnull + protected SiblingGraphService getInstance() { + return new SiblingGraphService(_entityService, graphService); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java index e98f38bba5de9d..154681f7fb467d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java @@ -24,7 +24,7 @@ public class ConfigurationProvider { */ private AuthenticationConfiguration authentication; /** - * Authentication related configs + * Authorizer related configs */ private AuthorizationConfiguration authorization; /** diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java index 75e2fa334e563e..84f76d21fdedba 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java @@ -2,11 +2,13 @@ import com.linkedin.entity.client.JavaEntityClient; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; +import com.linkedin.metadata.entity.DeleteEntityService; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -22,6 +24,10 @@ public class JavaEntityClientFactory { @Qualifier("entityService") private EntityService _entityService; + @Autowired + @Qualifier("deleteEntityService") + private DeleteEntityService _deleteEntityService; + @Autowired @Qualifier("searchService") private SearchService _searchService; @@ -30,6 +36,10 @@ public class JavaEntityClientFactory { @Qualifier("entitySearchService") private EntitySearchService _entitySearchService; + @Autowired + @Qualifier("cachingEntitySearchService") + private CachingEntitySearchService _cachingEntitySearchService; + @Autowired @Qualifier("timeseriesAspectService") private TimeseriesAspectService _timeseriesAspectService; @@ -46,10 +56,12 @@ public class JavaEntityClientFactory { public JavaEntityClient getJavaEntityClient() { return new JavaEntityClient( _entityService, - _eventProducer, + _deleteEntityService, _entitySearchService, + _cachingEntitySearchService, _searchService, + _lineageSearchService, _timeseriesAspectService, - _lineageSearchService); + _eventProducer); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index 79ff05a7c3da06..80c033018af2a7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -1,6 +1,8 @@ package com.linkedin.gms.factory.graphql; +import com.datahub.authentication.group.GroupService; import com.datahub.authentication.token.StatefulTokenService; +import com.datahub.authentication.user.NativeUserService; import com.linkedin.datahub.graphql.GmsGraphQLEngine; import com.linkedin.datahub.graphql.GraphQLEngine; import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; @@ -9,6 +11,7 @@ import com.linkedin.gms.factory.common.GitVersionFactory; import com.linkedin.gms.factory.common.IndexConventionFactory; import com.linkedin.gms.factory.common.RestHighLevelClientFactory; +import com.linkedin.gms.factory.common.SiblingGraphServiceFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.entity.RestliEntityClientFactory; @@ -16,6 +19,7 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.graph.SiblingGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; @@ -37,7 +41,7 @@ @Configuration @Import({RestHighLevelClientFactory.class, IndexConventionFactory.class, RestliEntityClientFactory.class, RecommendationServiceFactory.class, EntityRegistryFactory.class, DataHubTokenServiceFactory.class, - GitVersionFactory.class}) + GitVersionFactory.class, SiblingGraphServiceFactory.class}) public class GraphQLEngineFactory { @Autowired @Qualifier("elasticSearchRestHighLevelClient") @@ -67,6 +71,10 @@ public class GraphQLEngineFactory { @Qualifier("graphService") private GraphService _graphService; + @Autowired + @Qualifier("siblingGraphService") + private SiblingGraphService _siblingGraphService; + @Autowired @Qualifier("timeseriesAspectService") private TimeseriesAspectService _timeseriesAspectService; @@ -97,6 +105,14 @@ public class GraphQLEngineFactory { @Qualifier("timelineService") private TimelineService _timelineService; + @Autowired + @Qualifier("nativeUserService") + private NativeUserService _nativeUserService; + + @Autowired + @Qualifier("groupService") + private GroupService _groupService; + @Value("${platformAnalytics.enabled}") // TODO: Migrate to DATAHUB_ANALYTICS_ENABLED private Boolean isAnalyticsEnabled; @@ -115,6 +131,7 @@ protected GraphQLEngine getInstance() { _timeseriesAspectService, _entityRegistry, _secretService, + _nativeUserService, _configProvider.getIngestion(), _configProvider.getAuthentication(), _configProvider.getAuthorization(), @@ -124,7 +141,9 @@ protected GraphQLEngine getInstance() { _configProvider.getVisualConfig(), _configProvider.getTelemetry(), _configProvider.getMetadataTests(), - _configProvider.getDatahub() + _configProvider.getDatahub(), + _siblingGraphService, + _groupService ).builder().build(); } return new GmsGraphQLEngine( @@ -138,6 +157,7 @@ protected GraphQLEngine getInstance() { _timeseriesAspectService, _entityRegistry, _secretService, + _nativeUserService, _configProvider.getIngestion(), _configProvider.getAuthentication(), _configProvider.getAuthorization(), @@ -147,7 +167,9 @@ protected GraphQLEngine getInstance() { _configProvider.getVisualConfig(), _configProvider.getTelemetry(), _configProvider.getMetadataTests(), - _configProvider.getDatahub() + _configProvider.getDatahub(), + _siblingGraphService, + _groupService ).builder().build(); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/AllEntitiesSearchAggregatorFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/AllEntitiesSearchAggregatorFactory.java new file mode 100644 index 00000000000000..c3076dca2abb48 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/AllEntitiesSearchAggregatorFactory.java @@ -0,0 +1,48 @@ +package com.linkedin.gms.factory.search; + +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; +import com.linkedin.metadata.search.client.CachingEntitySearchService; +import com.linkedin.metadata.search.ranker.SearchRanker; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.PropertySource; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class AllEntitiesSearchAggregatorFactory { + + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry entityRegistry; + + @Autowired + @Qualifier("entitySearchService") + private EntitySearchService entitySearchService; + + @Autowired + @Qualifier("cachingEntitySearchService") + private CachingEntitySearchService cachingEntitySearchService; + + @Autowired + @Qualifier("searchRanker") + private SearchRanker searchRanker; + + @Bean(name = "allEntitiesSearchAggregator") + @Primary + @Nonnull + protected AllEntitiesSearchAggregator getInstance() { + return new AllEntitiesSearchAggregator( + entityRegistry, + entitySearchService, + cachingEntitySearchService, + searchRanker); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingAllEntitiesSearchAggregatorFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingAllEntitiesSearchAggregatorFactory.java new file mode 100644 index 00000000000000..d575369ca48f50 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingAllEntitiesSearchAggregatorFactory.java @@ -0,0 +1,44 @@ +package com.linkedin.gms.factory.search; + +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.search.aggregator.AllEntitiesSearchAggregator; +import com.linkedin.metadata.search.cache.CachingAllEntitiesSearchAggregator; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.CacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.PropertySource; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class CachingAllEntitiesSearchAggregatorFactory { + + @Autowired + @Qualifier("allEntitiesSearchAggregator") + private AllEntitiesSearchAggregator allEntitiesSearchAggregator; + + @Autowired + private CacheManager cacheManager; + + @Value("${searchService.resultBatchSize}") + private Integer batchSize; + + @Value("${searchService.enableCache}") + private Boolean enableCache; + + @Bean(name = "cachingAllEntitiesSearchAggregator") + @Primary + @Nonnull + protected CachingAllEntitiesSearchAggregator getInstance() { + return new CachingAllEntitiesSearchAggregator( + cacheManager, + allEntitiesSearchAggregator, + batchSize, + enableCache); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java new file mode 100644 index 00000000000000..7b20e798b79f26 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/CachingEntitySearchServiceFactory.java @@ -0,0 +1,44 @@ +package com.linkedin.gms.factory.search; + +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.client.CachingEntitySearchService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.CacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.PropertySource; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class CachingEntitySearchServiceFactory { + + @Autowired + @Qualifier("entitySearchService") + private EntitySearchService entitySearchService; + + @Autowired + private CacheManager cacheManager; + + @Value("${searchService.resultBatchSize}") + private Integer batchSize; + + @Value("${searchService.enableCache}") + private Boolean enableCache; + + @Bean(name = "cachingEntitySearchService") + @Primary + @Nonnull + protected CachingEntitySearchService getInstance() { + return new CachingEntitySearchService( + cacheManager, + entitySearchService, + batchSize, + enableCache); + } +} \ No newline at end of file diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java index 8fd6641d47ee97..1ead5669fdc566 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SearchServiceFactory.java @@ -4,12 +4,13 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.cache.CachingAllEntitiesSearchAggregator; +import com.linkedin.metadata.search.cache.EntityDocCountCache; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.search.ranker.SearchRanker; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -29,22 +30,25 @@ public class SearchServiceFactory { private EntitySearchService entitySearchService; @Autowired - @Qualifier("searchRanker") - private SearchRanker searchRanker; + @Qualifier("cachingEntitySearchService") + private CachingEntitySearchService cachingEntitySearchService; @Autowired - private CacheManager cacheManager; - - @Value("${searchService.resultBatchSize}") - private Integer batchSize; + @Qualifier("cachingAllEntitiesSearchAggregator") + private CachingAllEntitiesSearchAggregator cachingAllEntitiesSearchAggregator; - @Value("${searchService.enableCache}") - private Boolean enableCache; + @Autowired + @Qualifier("searchRanker") + private SearchRanker searchRanker; @Bean(name = "searchService") @Primary @Nonnull protected SearchService getInstance() { - return new SearchService(entityRegistry, entitySearchService, searchRanker, cacheManager, batchSize, enableCache); + return new SearchService( + new EntityDocCountCache(entityRegistry, entitySearchService), + cachingEntitySearchService, + cachingAllEntitiesSearchAggregator, + searchRanker); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManager.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManager.java index 6e3dba25656b58..ddcd05b3a8aaa8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManager.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManager.java @@ -20,12 +20,14 @@ public BootstrapManager(final List bootSteps) { } public void start() { - // Once the application has been set up, apply boot steps. log.info("Starting Bootstrap Process..."); - for (int i = 0; i < _bootSteps.size(); i++) { - final BootstrapStep step = _bootSteps.get(i); + + List stepsToExecute = _bootSteps; + + for (int i = 0; i < stepsToExecute.size(); i++) { + final BootstrapStep step = stepsToExecute.get(i); if (step.getExecutionMode() == BootstrapStep.ExecutionMode.BLOCKING) { - log.info("Executing bootstrap step {}/{} with name {}...", i + 1, _bootSteps.size(), step.name()); + log.info("Executing bootstrap step {}/{} with name {}...", i + 1, stepsToExecute.size(), step.name()); try { step.execute(); } catch (Exception e) { @@ -33,12 +35,12 @@ public void start() { System.exit(1); } } else { // Async - log.info("Starting asynchronous bootstrap step {}/{} with name {}...", i + 1, _bootSteps.size(), step.name()); + log.info("Starting asynchronous bootstrap step {}/{} with name {}...", i + 1, stepsToExecute.size(), step.name()); CompletableFuture.runAsync(() -> { try { step.execute(); } catch (Exception e) { - log.error(String.format("Caught exception while executing bootstrap step %s. Exiting...", step.name()), e); + log.error(String.format("Caught exception while executing bootstrap step %s. Continuing...", step.name()), e); } }); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java index c53cdd6d8b6eb2..df6cd9572dc047 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java @@ -30,6 +30,6 @@ enum ExecutionMode { // Block service from starting up while running the step BLOCKING, // Start the step asynchronously without waiting for it to end - ASYNC; + ASYNC, } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManagerApplicationListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java similarity index 83% rename from metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManagerApplicationListener.java rename to metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java index 3f13a58f60d155..5e496c04c54f67 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapManagerApplicationListener.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java @@ -11,11 +11,11 @@ /** - * Responsible for coordinating boot-time checks. + * Responsible for coordinating starting steps that happen before the application starts up. */ @Slf4j @Component -public class BootstrapManagerApplicationListener implements ApplicationListener { +public class OnBootApplicationListener implements ApplicationListener { private static final String ROOT_WEB_APPLICATION_CONTEXT_ID = String.format("%s:", WebApplicationContext.class.getName()); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java new file mode 100644 index 00000000000000..3cf05658927cae --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java @@ -0,0 +1,125 @@ +package com.linkedin.metadata.boot; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.entity.EntityResponse; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.key.DataHubUpgradeKey; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.upgrade.DataHubUpgradeRequest; +import com.linkedin.upgrade.DataHubUpgradeResult; +import java.net.URISyntaxException; +import java.util.Collections; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public abstract class UpgradeStep implements BootstrapStep { + private static final Integer SLEEP_SECONDS = 120; + + protected final EntityService _entityService; + private final String _version; + private final String _upgradeId; + private final Urn _upgradeUrn; + + public UpgradeStep(EntityService entityService, String version, String upgradeId) { + this._entityService = entityService; + this._version = version; + this._upgradeId = upgradeId; + this._upgradeUrn = EntityKeyUtils.convertEntityKeyToUrn(new DataHubUpgradeKey().setId(upgradeId), + Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + } + + @Override + public void execute() throws Exception { + String upgradeStepName = name(); + + log.info(String.format("Attempting to run %s Upgrade Step..", upgradeStepName)); + log.info(String.format("Waiting %s seconds..", SLEEP_SECONDS)); + + if (hasUpgradeRan()) { + log.info(String.format("%s has run before for version %s. Skipping..", _upgradeId, _version)); + return; + } + + // Sleep to ensure deployment process finishes. + Thread.sleep(SLEEP_SECONDS * 1000); + + try { + ingestUpgradeRequestAspect(); + upgrade(); + ingestUpgradeResultAspect(); + } catch (Exception e) { + String errorMessage = String.format("Error when running %s for version %s", _upgradeId, _version); + cleanUpgradeAfterError(e, errorMessage); + throw new RuntimeException(errorMessage, e); + } + } + + @Override + public String name() { + return this.getClass().getSimpleName(); + } + + public abstract void upgrade() throws Exception; + + private boolean hasUpgradeRan() { + try { + EntityResponse response = _entityService.getEntityV2(Constants.DATA_HUB_UPGRADE_ENTITY_NAME, _upgradeUrn, + Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)); + + if (response != null && response.getAspects().containsKey(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)) { + DataMap dataMap = response.getAspects().get(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME).getValue().data(); + DataHubUpgradeRequest request = new DataHubUpgradeRequest(dataMap); + if (request.hasVersion() && request.getVersion().equals(_version)) { + return true; + } + } + } catch (Exception e) { + log.error("Error when checking to see if datahubUpgrade entity exists. Commencing with upgrade...", e); + return false; + } + return false; + } + + private void ingestUpgradeRequestAspect() throws URISyntaxException { + final AuditStamp auditStamp = + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + final DataHubUpgradeRequest upgradeRequest = + new DataHubUpgradeRequest().setTimestampMs(System.currentTimeMillis()).setVersion(_version); + + final MetadataChangeProposal upgradeProposal = new MetadataChangeProposal(); + upgradeProposal.setEntityUrn(_upgradeUrn); + upgradeProposal.setEntityType(Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + upgradeProposal.setAspectName(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME); + upgradeProposal.setAspect(GenericRecordUtils.serializeAspect(upgradeRequest)); + upgradeProposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal(upgradeProposal, auditStamp); + } + + private void ingestUpgradeResultAspect() throws URISyntaxException { + final AuditStamp auditStamp = + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + final DataHubUpgradeResult upgradeResult = new DataHubUpgradeResult().setTimestampMs(System.currentTimeMillis()); + + final MetadataChangeProposal upgradeProposal = new MetadataChangeProposal(); + upgradeProposal.setEntityUrn(_upgradeUrn); + upgradeProposal.setEntityType(Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + upgradeProposal.setAspectName(Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME); + upgradeProposal.setAspect(GenericRecordUtils.serializeAspect(upgradeResult)); + upgradeProposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal(upgradeProposal, auditStamp); + } + + private void cleanUpgradeAfterError(Exception e, String errorMessage) { + log.error(errorMessage, e); + _entityService.deleteUrn(_upgradeUrn); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java index 058ae92da7c649..d092ce88176508 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java @@ -11,12 +11,15 @@ import com.linkedin.metadata.boot.steps.IngestPoliciesStep; import com.linkedin.metadata.boot.steps.IngestRetentionPoliciesStep; import com.linkedin.metadata.boot.steps.IngestRootUserStep; +import com.linkedin.metadata.boot.steps.RemoveClientIdAspectStep; +import com.linkedin.metadata.boot.steps.RestoreDbtSiblingsIndices; import com.linkedin.metadata.boot.steps.RestoreGlossaryIndices; import com.linkedin.metadata.entity.AspectMigrationsDao; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; +import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -24,8 +27,6 @@ import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Scope; -import javax.annotation.Nonnull; - @Configuration @Import({EntityServiceFactory.class, EntityRegistryFactory.class, EntitySearchServiceFactory.class, @@ -66,8 +67,14 @@ protected BootstrapManager createInstance() { final IngestDataPlatformsStep ingestDataPlatformsStep = new IngestDataPlatformsStep(_entityService); final IngestDataPlatformInstancesStep ingestDataPlatformInstancesStep = new IngestDataPlatformInstancesStep(_entityService, _migrationsDao); - final RestoreGlossaryIndices restoreGlossaryIndicesStep = new RestoreGlossaryIndices(_entityService, _entitySearchService, _entityRegistry); - return new BootstrapManager(ImmutableList.of(ingestRootUserStep, ingestPoliciesStep, ingestDataPlatformsStep, - ingestDataPlatformInstancesStep, _ingestRetentionPoliciesStep, restoreGlossaryIndicesStep)); + final RestoreGlossaryIndices restoreGlossaryIndicesStep = + new RestoreGlossaryIndices(_entityService, _entitySearchService, _entityRegistry); + final RestoreDbtSiblingsIndices restoreDbtSiblingsIndices = + new RestoreDbtSiblingsIndices(_entityService, _entityRegistry); + final RemoveClientIdAspectStep removeClientIdAspectStep = new RemoveClientIdAspectStep(_entityService); + return new BootstrapManager( + ImmutableList.of(ingestRootUserStep, ingestPoliciesStep, ingestDataPlatformsStep, + ingestDataPlatformInstancesStep, _ingestRetentionPoliciesStep, restoreGlossaryIndicesStep, + removeClientIdAspectStep, restoreDbtSiblingsIndices)); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java index 53b7d241d03f4e..8dfaced5d6a880 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java @@ -17,11 +17,13 @@ import javax.annotation.Nonnull; import java.util.Optional; +import static com.linkedin.metadata.Constants.*; + @Slf4j @RequiredArgsConstructor public class IngestDataPlatformInstancesStep implements BootstrapStep { - private static final String PLATFORM_INSTANCE_ASPECT_NAME = "dataPlatformInstance"; + private static final int BATCH_SIZE = 1000; private final EntityService _entityService; @@ -47,7 +49,7 @@ private Optional getDataPlatformInstance(Urn urn) { @Override public void execute() throws Exception { log.info("Checking for DataPlatformInstance"); - if (_migrationsDao.checkIfAspectExists(PLATFORM_INSTANCE_ASPECT_NAME)) { + if (_migrationsDao.checkIfAspectExists(DATA_PLATFORM_INSTANCE_ASPECT_NAME)) { log.info("DataPlatformInstance aspect exists. Skipping step"); return; } @@ -69,7 +71,7 @@ public void execute() throws Exception { final AuditStamp aspectAuditStamp = new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); - _entityService.ingestAspect(urn, PLATFORM_INSTANCE_ASPECT_NAME, dataPlatformInstance.get(), aspectAuditStamp, null); + _entityService.ingestAspect(urn, DATA_PLATFORM_INSTANCE_ASPECT_NAME, dataPlatformInstance.get(), aspectAuditStamp, null); } log.info("Finished ingesting DataPlatformInstance for urn {} to {}", start, start + BATCH_SIZE); start += BATCH_SIZE; diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java index 56910cf24baea0..7d460419adf7b2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.boot.steps; +import com.datahub.util.RecordUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.AuditStamp; @@ -7,12 +8,9 @@ import com.linkedin.dataplatform.DataPlatformInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.BootstrapStep; -import com.datahub.util.RecordUtils; import com.linkedin.metadata.entity.EntityService; - import java.io.IOException; import java.net.URISyntaxException; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.ClassPathResource; @@ -46,13 +44,24 @@ public void execute() throws IOException, URISyntaxException { // 2. For each JSON object, cast into a DataPlatformSnapshot object. for (final JsonNode dataPlatform : dataPlatforms) { + final String urnString; final Urn urn; try { - urn = Urn.createFromString(dataPlatform.get("urn").asText()); + urnString = dataPlatform.get("urn").asText(); + urn = Urn.createFromString(urnString); } catch (URISyntaxException e) { log.error("Malformed urn: {}", dataPlatform.get("urn").asText()); throw new RuntimeException("Malformed urn", e); } + + final DataPlatformInfo existingInfo = + (DataPlatformInfo) _entityService.getLatestAspect(urn, PLATFORM_ASPECT_NAME); + // Skip ingesting for this JSON object if info already exists. + if (existingInfo != null) { + log.debug(String.format("%s already exists for %s. Skipping...", PLATFORM_ASPECT_NAME, urnString)); + continue; + } + final DataPlatformInfo info = RecordUtils.toRecordTemplate(DataPlatformInfo.class, dataPlatform.get("aspect").toString()); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RemoveClientIdAspectStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RemoveClientIdAspectStep.java new file mode 100644 index 00000000000000..b76d935a060227 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RemoveClientIdAspectStep.java @@ -0,0 +1,76 @@ +package com.linkedin.metadata.boot.steps; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.gms.factory.telemetry.TelemetryUtils; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.key.DataHubUpgradeKey; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.upgrade.DataHubUpgradeResult; +import java.util.HashMap; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +@RequiredArgsConstructor +public class RemoveClientIdAspectStep implements BootstrapStep { + + private final EntityService _entityService; + + private static final String VERSION = "0"; + private static final String UPGRADE_ID = "remove-unknown-aspects"; + private static final String INVALID_TELEMETRY_ASPECT_NAME = "clientId"; + private static final Urn REMOVE_UNKNOWN_ASPECTS_URN = + EntityKeyUtils.convertEntityKeyToUrn(new DataHubUpgradeKey().setId(UPGRADE_ID), Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + + @Override + public String name() { + return this.getClass().getSimpleName(); + } + + @Override + public void execute() throws Exception { + try { + if (_entityService.exists(REMOVE_UNKNOWN_ASPECTS_URN)) { + log.info("Unknown aspects have been removed. Skipping..."); + return; + } + // Remove invalid telemetry aspect + _entityService.deleteAspect(TelemetryUtils.CLIENT_ID_URN, INVALID_TELEMETRY_ASPECT_NAME, new HashMap<>(), true); + + final AuditStamp auditStamp = new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + final DataHubUpgradeResult upgradeResult = new DataHubUpgradeResult().setTimestampMs(System.currentTimeMillis()); + ingestUpgradeAspect(Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, upgradeResult, auditStamp); + } catch (Exception e) { + log.error("Error when running the RemoveUnknownAspects Bootstrap Step", e); + _entityService.deleteUrn(REMOVE_UNKNOWN_ASPECTS_URN); + throw new RuntimeException("Error when running the RemoveUnknownAspects Bootstrap Step", e); + } + } + + @Nonnull + @Override + public ExecutionMode getExecutionMode() { + return ExecutionMode.ASYNC; + } + + private void ingestUpgradeAspect(String aspectName, RecordTemplate aspect, AuditStamp auditStamp) { + final MetadataChangeProposal upgradeProposal = new MetadataChangeProposal(); + upgradeProposal.setEntityUrn(REMOVE_UNKNOWN_ASPECTS_URN); + upgradeProposal.setEntityType(Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + upgradeProposal.setAspectName(aspectName); + upgradeProposal.setAspect(GenericRecordUtils.serializeAspect(aspect)); + upgradeProposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal(upgradeProposal, auditStamp); + } + +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java new file mode 100644 index 00000000000000..374ff716ec2ccc --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java @@ -0,0 +1,173 @@ +package com.linkedin.metadata.boot.steps; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.dataset.UpstreamLineage; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.key.DataHubUpgradeKey; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.ListUrnsResult; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.upgrade.DataHubUpgradeRequest; +import com.linkedin.upgrade.DataHubUpgradeResult; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.Constants.*; + + +@Slf4j +@RequiredArgsConstructor +public class RestoreDbtSiblingsIndices implements BootstrapStep { + private static final String VERSION = "0"; + private static final String UPGRADE_ID = "restore-dbt-siblings-indices"; + private static final Urn SIBLING_UPGRADE_URN = + EntityKeyUtils.convertEntityKeyToUrn(new DataHubUpgradeKey().setId(UPGRADE_ID), Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + private static final Integer BATCH_SIZE = 1000; + private static final Integer SLEEP_SECONDS = 120; + + private final EntityService _entityService; + private final EntityRegistry _entityRegistry; + + @Override + public String name() { + return this.getClass().getSimpleName(); + } + + @Nonnull + @Override + public ExecutionMode getExecutionMode() { + return ExecutionMode.ASYNC; + } + + @Override + public void execute() throws Exception { + log.info("Attempting to run RestoreDbtSiblingsIndices upgrade.."); + log.info(String.format("Waiting %s seconds..", SLEEP_SECONDS)); + + // Sleep to ensure deployment process finishes. + Thread.sleep(SLEEP_SECONDS * 1000); + + EntityResponse response = _entityService.getEntityV2( + Constants.DATA_HUB_UPGRADE_ENTITY_NAME, SIBLING_UPGRADE_URN, + Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME) + ); + if (response != null && response.getAspects().containsKey(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME)) { + DataMap dataMap = response.getAspects().get(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME).getValue().data(); + DataHubUpgradeRequest request = new DataHubUpgradeRequest(dataMap); + if (request.hasVersion() && request.getVersion().equals(VERSION)) { + log.info("RestoreDbtSiblingsIndices has run before with this version. Skipping"); + return; + } + } + + log.info("Bootstrapping sibling aspects"); + + try { + final int rowCount = _entityService.listUrns(DATASET_ENTITY_NAME, 0, 10).getTotal(); + + log.info("Found {} dataset entities to attempt to bootstrap", rowCount); + + final AspectSpec datasetAspectSpec = + _entityRegistry.getEntitySpec(Constants.DATASET_ENTITY_NAME).getAspectSpec(Constants.UPSTREAM_LINEAGE_ASPECT_NAME); + final AuditStamp auditStamp = new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + + final DataHubUpgradeRequest upgradeRequest = new DataHubUpgradeRequest().setTimestampMs(System.currentTimeMillis()).setVersion(VERSION); + ingestUpgradeAspect(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, upgradeRequest, auditStamp); + + int indexedCount = 0; + while (indexedCount < rowCount) { + getAndRestoreUpstreamLineageIndices(indexedCount, auditStamp, datasetAspectSpec); + indexedCount += BATCH_SIZE; + } + + final DataHubUpgradeResult upgradeResult = new DataHubUpgradeResult().setTimestampMs(System.currentTimeMillis()); + ingestUpgradeAspect(Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, upgradeResult, auditStamp); + + log.info("Successfully restored sibling aspects"); + } catch (Exception e) { + log.error("Error when running the RestoreDbtSiblingsIndices Bootstrap Step", e); + _entityService.deleteUrn(SIBLING_UPGRADE_URN); + throw new RuntimeException("Error when running the RestoreDbtSiblingsIndices Bootstrap Step", e); + } + } + + private void getAndRestoreUpstreamLineageIndices(int start, AuditStamp auditStamp, AspectSpec upstreamAspectSpec) { + ListUrnsResult datasetUrnsResult = _entityService.listUrns(DATASET_ENTITY_NAME, start, BATCH_SIZE); + List datasetUrns = datasetUrnsResult.getEntities(); + log.info("Re-indexing upstreamLineage aspect from {} with batch size {}", start, BATCH_SIZE); + + if (datasetUrns.size() == 0) { + return; + } + + final Map upstreamLineageResponse; + try { + upstreamLineageResponse = + _entityService.getEntitiesV2(DATASET_ENTITY_NAME, new HashSet<>(datasetUrns), Collections.singleton(UPSTREAM_LINEAGE_ASPECT_NAME)); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Error fetching upstream lineage history: %s", e.toString())); + } + + // Loop over datasets and produce changelog + for (Urn datasetUrn : datasetUrns) { + EntityResponse response = upstreamLineageResponse.get(datasetUrn); + if (response == null) { + log.warn("Dataset not in set of entity responses {}", datasetUrn); + continue; + } + UpstreamLineage upstreamLineage = getUpstreamLineage(response); + if (upstreamLineage == null) { + continue; + } + + _entityService.produceMetadataChangeLog( + datasetUrn, + DATASET_ENTITY_NAME, + UPSTREAM_LINEAGE_ASPECT_NAME, + upstreamAspectSpec, + null, + upstreamLineage, + null, + null, + auditStamp, + ChangeType.RESTATE); + } + } + + private UpstreamLineage getUpstreamLineage(EntityResponse entityResponse) { + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + if (!aspectMap.containsKey(UPSTREAM_LINEAGE_ASPECT_NAME)) { + return null; + } + + return new UpstreamLineage(aspectMap.get(Constants.UPSTREAM_LINEAGE_ASPECT_NAME).getValue().data()); + } + + private void ingestUpgradeAspect(String aspectName, RecordTemplate aspect, AuditStamp auditStamp) { + final MetadataChangeProposal upgradeProposal = new MetadataChangeProposal(); + upgradeProposal.setEntityUrn(SIBLING_UPGRADE_URN); + upgradeProposal.setEntityType(Constants.DATA_HUB_UPGRADE_ENTITY_NAME); + upgradeProposal.setAspectName(aspectName); + upgradeProposal.setAspect(GenericRecordUtils.serializeAspect(aspect)); + upgradeProposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal(upgradeProposal, auditStamp); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java index a932c9e284fc55..b5ae5fb34a434b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java @@ -2,129 +2,89 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; -import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.boot.UpgradeStep; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.key.DataHubUpgradeKey; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; -import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; -import com.linkedin.upgrade.DataHubUpgradeRequest; -import com.linkedin.upgrade.DataHubUpgradeResult; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; @Slf4j -@RequiredArgsConstructor -public class RestoreGlossaryIndices implements BootstrapStep { - private static final String VERSION = "0"; +public class RestoreGlossaryIndices extends UpgradeStep { + private static final String VERSION = "1"; private static final String UPGRADE_ID = "restore-glossary-indices-ui"; - private static final Urn GLOSSARY_UPGRADE_URN = - EntityKeyUtils.convertEntityKeyToUrn(new DataHubUpgradeKey().setId(UPGRADE_ID), Constants.DATA_HUB_UPGRADE_ENTITY_NAME); private static final Integer BATCH_SIZE = 1000; - private final EntityService _entityService; private final EntitySearchService _entitySearchService; private final EntityRegistry _entityRegistry; - @Override - public String name() { - return this.getClass().getSimpleName(); - } - - @Nonnull - @Override - public ExecutionMode getExecutionMode() { - return ExecutionMode.BLOCKING; + public RestoreGlossaryIndices(EntityService entityService, EntitySearchService entitySearchService, + EntityRegistry entityRegistry) { + super(entityService, VERSION, UPGRADE_ID); + _entitySearchService = entitySearchService; + _entityRegistry = entityRegistry; } @Override - public void execute() throws Exception { - log.info("Attempting to run RestoreGlossaryIndices upgrade.."); - try { - if (_entityService.exists(GLOSSARY_UPGRADE_URN)) { - log.info("Glossary Upgrade has run before. Skipping"); - return; - } - - final AspectSpec termAspectSpec = - _entityRegistry.getEntitySpec(Constants.GLOSSARY_TERM_ENTITY_NAME).getAspectSpec(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME); - final AspectSpec nodeAspectSpec = - _entityRegistry.getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME).getAspectSpec(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME); - final AuditStamp auditStamp = new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); - - final DataHubUpgradeRequest upgradeRequest = new DataHubUpgradeRequest().setTimestampMs(System.currentTimeMillis()).setVersion(VERSION); - ingestUpgradeAspect(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, upgradeRequest, auditStamp); - - final int totalTermsCount = getAndRestoreTermAspectIndices(0, auditStamp, termAspectSpec); - int termsCount = BATCH_SIZE; - while (termsCount < totalTermsCount) { - getAndRestoreTermAspectIndices(termsCount, auditStamp, termAspectSpec); - termsCount += BATCH_SIZE; - } - - final int totalNodesCount = getAndRestoreNodeAspectIndices(0, auditStamp, nodeAspectSpec); - int nodesCount = BATCH_SIZE; - while (nodesCount < totalNodesCount) { - getAndRestoreNodeAspectIndices(nodesCount, auditStamp, nodeAspectSpec); - nodesCount += BATCH_SIZE; - } - - final DataHubUpgradeResult upgradeResult = new DataHubUpgradeResult().setTimestampMs(System.currentTimeMillis()); - ingestUpgradeAspect(Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, upgradeResult, auditStamp); + public void upgrade() throws Exception { + final AspectSpec termAspectSpec = _entityRegistry.getEntitySpec(Constants.GLOSSARY_TERM_ENTITY_NAME) + .getAspectSpec(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME); + final AspectSpec nodeAspectSpec = _entityRegistry.getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME) + .getAspectSpec(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME); + final AuditStamp auditStamp = + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()); + + final int totalTermsCount = getAndRestoreTermAspectIndices(0, auditStamp, termAspectSpec); + int termsCount = BATCH_SIZE; + while (termsCount < totalTermsCount) { + getAndRestoreTermAspectIndices(termsCount, auditStamp, termAspectSpec); + termsCount += BATCH_SIZE; + } - log.info("Successfully restored glossary index"); - } catch (Exception e) { - log.error("Error when running the RestoreGlossaryIndices Bootstrap Step", e); - _entityService.deleteUrn(GLOSSARY_UPGRADE_URN); - throw new RuntimeException("Error when running the RestoreGlossaryIndices Bootstrap Step", e); + final int totalNodesCount = getAndRestoreNodeAspectIndices(0, auditStamp, nodeAspectSpec); + int nodesCount = BATCH_SIZE; + while (nodesCount < totalNodesCount) { + getAndRestoreNodeAspectIndices(nodesCount, auditStamp, nodeAspectSpec); + nodesCount += BATCH_SIZE; } } - private void ingestUpgradeAspect(String aspectName, RecordTemplate aspect, AuditStamp auditStamp) { - final MetadataChangeProposal upgradeProposal = new MetadataChangeProposal(); - upgradeProposal.setEntityUrn(GLOSSARY_UPGRADE_URN); - upgradeProposal.setEntityType(Constants.DATA_HUB_UPGRADE_ENTITY_NAME); - upgradeProposal.setAspectName(aspectName); - upgradeProposal.setAspect(GenericRecordUtils.serializeAspect(aspect)); - upgradeProposal.setChangeType(ChangeType.UPSERT); - - _entityService.ingestProposal(upgradeProposal, auditStamp); + @Nonnull + @Override + public ExecutionMode getExecutionMode() { + return ExecutionMode.ASYNC; } - private int getAndRestoreTermAspectIndices(int start, AuditStamp auditStamp, AspectSpec termAspectSpec) throws Exception { - SearchResult termsResult = _entitySearchService.search(Constants.GLOSSARY_TERM_ENTITY_NAME, "", null, null, start, BATCH_SIZE); + private int getAndRestoreTermAspectIndices(int start, AuditStamp auditStamp, AspectSpec termAspectSpec) + throws Exception { + SearchResult termsResult = + _entitySearchService.search(Constants.GLOSSARY_TERM_ENTITY_NAME, "", null, null, start, BATCH_SIZE); List termUrns = termsResult.getEntities().stream().map(SearchEntity::getEntity).collect(Collectors.toList()); if (termUrns.size() == 0) { return 0; } - final Map termInfoResponses = _entityService.getEntitiesV2( - Constants.GLOSSARY_TERM_ENTITY_NAME, - new HashSet<>(termUrns), + final Map termInfoResponses = + _entityService.getEntitiesV2(Constants.GLOSSARY_TERM_ENTITY_NAME, new HashSet<>(termUrns), Collections.singleton(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME) ); // Loop over Terms and produce changelog - for (Urn termUrn: termUrns) { + for (Urn termUrn : termUrns) { EntityResponse termEntityResponse = termInfoResponses.get(termUrn); if (termEntityResponse == null) { log.warn("Term not in set of entity responses {}", termUrn); @@ -165,7 +125,7 @@ private int getAndRestoreNodeAspectIndices(int start, AuditStamp auditStamp, Asp ); // Loop over Nodes and produce changelog - for (Urn nodeUrn: nodeUrns) { + for (Urn nodeUrn : nodeUrns) { EntityResponse nodeEntityResponse = nodeInfoResponses.get(nodeUrn); if (nodeEntityResponse == null) { log.warn("Node not in set of entity responses {}", nodeUrn); diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index 4074dcd2ff4e00..6ccd9d7511723b 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -36,13 +36,15 @@ authorization: # Optional: A set of custom authorizers, serving in addition to the default DataHub policies-based authorizer. authorizers: - # - type: com.datahub.authorization.authorizer.CustomAuthorizer - # configs: - # config1: value1 + - type: com.datahub.authorization.ranger.RangerAuthorizer + enabled: ${RANGER_AUTHORIZER_ENABLED:false} + configs: + username: ${RANGER_USERNAME} + password: ${RANGER_PASSWORD} ingestion: enabled: ${UI_INGESTION_ENABLED:true} - defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:0.8.35}' + defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:0.8.42}' telemetry: enabledCli: ${CLI_TELEMETRY_ENABLED:true} @@ -56,8 +58,8 @@ secretService: datahub: serverType: ${DATAHUB_SERVER_TYPE:prod} gms: - host: ${DATAHUB_GMS_HOST:${GMS_HOST:localhost}} - port: ${DATAHUB_GMS_PORT:${GMS_PORT:8080}} + host: ${DATAHUB_GMS_HOST:localhost} + port: ${DATAHUB_GMS_PORT:8080} useSSL: ${DATAHUB_GMS_USE_SSL:${GMS_USE_SSL:false}} sslContext: protocol: ${DATAHUB_GMS_SSL_PROTOCOL:${GMS_SSL_PROTOCOL:#{null}}} @@ -85,7 +87,7 @@ configEntityRegistry: path: ${ENTITY_REGISTRY_CONFIG_PATH:../../metadata-models/src/main/resources/entity-registry.yml} platformAnalytics: - enabled: ${ANALYTICS_ENABLED:true} + enabled: ${DATAHUB_ANALYTICS_ENABLED:true} visualConfig: assets: @@ -145,7 +147,7 @@ elasticsearch: prefix: ${INDEX_PREFIX:} numShards: ${ELASTICSEARCH_NUM_SHARDS_PER_INDEX:1} numReplicas: ${ELASTICSEARCH_NUM_REPLICAS_PER_INDEX:1} - numRetries: ${ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES :3} + numRetries: ${ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES:3} maxArrayLength: ${SEARCH_DOCUMENT_MAX_ARRAY_LENGTH:1000} mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}} @@ -160,6 +162,10 @@ kafka: awsGlue: region: ${AWS_GLUE_SCHEMA_REGISTRY_REGION:us-east-1} registryName: ${AWS_GLUE_SCHEMA_REGISTRY_NAME:#{null}} + schema: + registry: + security: + protocol: ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:PLAINTEXT} # Only required if GraphService type is neo4j neo4j: @@ -176,10 +182,16 @@ spring: mvc: servlet: path: /openapi + kafka: + security: + protocol: ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:PLAINTEXT} springdoc: cache: disabled: true metadataTests: - enabled: ${METADATA_TESTS_ENABLED:true} \ No newline at end of file + enabled: ${METADATA_TESTS_ENABLED:true} + +siblings: + enabled: ${ENABLE_SIBLING_HOOK:true} # enable to turn on automatic sibling associations for dbt diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java new file mode 100644 index 00000000000000..34fee379ed09ba --- /dev/null +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java @@ -0,0 +1,149 @@ +package com.linkedin.metadata.boot.steps; + +import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.entity.AspectMigrationsDao; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.utils.DataPlatformInstanceUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jetbrains.annotations.NotNull; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; + + +/** + * Test the behavior of IngestDataPlatformInstancesStep. + * + * We expect it to check if any data platform instance aspects already exist in the database and if none are found, + * to go through all the stored entities and ingest a data platform instance aspect for any that are compatible with it. + * + * CorpUser is used as an example of an entity that is not compatible with data platform instance and therefore should be ignored. + * Char is used as an example of an entity that should get adorned with a data platform instance. + * + * See {@link DataPlatformInstanceUtils} for the compatibility rules. + */ +public class IngestDataPlatformInstancesStepTest { + + @Test + public void testExecuteDoesNothingWhenDataPlatformInstanceAspectsAlreadyExists() throws Exception { + final EntityService entityService = mock(EntityService.class); + final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); + + mockDBWithDataPlatformInstanceAspects(migrationsDao); + + final IngestDataPlatformInstancesStep step = new IngestDataPlatformInstancesStep(entityService, migrationsDao); + step.execute(); + + verify(migrationsDao, times(1)).checkIfAspectExists(anyString()); + verifyNoMoreInteractions(migrationsDao); + verifyZeroInteractions(entityService); + } + + @Test + public void testExecuteCopesWithEmptyDB() throws Exception { + final EntityService entityService = mock(EntityService.class); + final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); + + mockEmptyDB(migrationsDao); + + final IngestDataPlatformInstancesStep step = new IngestDataPlatformInstancesStep(entityService, migrationsDao); + step.execute(); + + verify(migrationsDao, times(1)).checkIfAspectExists(anyString()); + verify(migrationsDao, times(1)).countEntities(); + verifyNoMoreInteractions(migrationsDao); + verifyZeroInteractions(entityService); + } + + @Test + public void testExecuteChecksKeySpecForAllUrns() throws Exception { + final EntityRegistry entityRegistry = getTestEntityRegistry(); + final EntityService entityService = mock(EntityService.class); + final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); + final int countOfCorpUserEntities = 2; + final int countOfChartEntities = 4; + final int totalUrnsInDB = countOfCorpUserEntities + countOfChartEntities; + + mockDBWithWorkToDo(entityRegistry, entityService, migrationsDao, countOfCorpUserEntities, countOfChartEntities); + + final IngestDataPlatformInstancesStep step = new IngestDataPlatformInstancesStep(entityService, migrationsDao); + step.execute(); + + verify(entityService, times(totalUrnsInDB)).getKeyAspectSpec(any(Urn.class)); + } + + @Test + public void testExecuteWhenSomeEntitiesShouldReceiveDataPlatformInstance() throws Exception { + final EntityRegistry entityRegistry = getTestEntityRegistry(); + final EntityService entityService = mock(EntityService.class); + final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); + final int countOfCorpUserEntities = 5; + final int countOfChartEntities = 7; + + mockDBWithWorkToDo(entityRegistry, entityService, migrationsDao, countOfCorpUserEntities, countOfChartEntities); + + final IngestDataPlatformInstancesStep step = new IngestDataPlatformInstancesStep(entityService, migrationsDao); + step.execute(); + + verify(entityService, times(countOfChartEntities)) + .ingestAspect( + argThat(arg -> arg.getEntityType().equals("chart")), + eq(DATA_PLATFORM_INSTANCE_ASPECT_NAME), + any(DataPlatformInstance.class), + any(), + any()); + verify(entityService, times(0)) + .ingestAspect(argThat(arg -> !arg.getEntityType().equals("chart")), anyString(), any(), any(), any()); + } + + @NotNull + private ConfigEntityRegistry getTestEntityRegistry() { + return new ConfigEntityRegistry( + IngestDataPlatformInstancesStepTest.class.getClassLoader().getResourceAsStream("test-entity-registry.yaml")); + } + + private void mockDBWithDataPlatformInstanceAspects(AspectMigrationsDao migrationsDao) { + when(migrationsDao.checkIfAspectExists(DATA_PLATFORM_INSTANCE_ASPECT_NAME)).thenReturn(true); + } + + private void mockEmptyDB(AspectMigrationsDao migrationsDao) { + when(migrationsDao.checkIfAspectExists(DATA_PLATFORM_INSTANCE_ASPECT_NAME)).thenReturn(false); + when(migrationsDao.countEntities()).thenReturn(0L); + } + + private void mockDBWithWorkToDo( + EntityRegistry entityRegistry, + EntityService entityService, + AspectMigrationsDao migrationsDao, + int countOfCorpUserEntities, + int countOfChartEntities) { + List corpUserUrns = insertMockEntities(countOfCorpUserEntities, "corpuser", "urn:li:corpuser:test%d", entityRegistry, entityService); + List charUrns = insertMockEntities(countOfChartEntities, "chart", "urn:li:chart:(looker,test%d)", entityRegistry, entityService); + List allUrnsInDB = Stream.concat(corpUserUrns.stream(), charUrns.stream()).map(Urn::toString).collect(Collectors.toList()); + when(migrationsDao.checkIfAspectExists(DATA_PLATFORM_INSTANCE_ASPECT_NAME)).thenReturn(false); + when(migrationsDao.countEntities()).thenReturn((long) allUrnsInDB.size()); + when(migrationsDao.listAllUrns(anyInt(), anyInt())).thenReturn(allUrnsInDB); + } + + private List insertMockEntities(int count, String entity, String urnTemplate, EntityRegistry entityRegistry, EntityService entityService) { + EntitySpec entitySpec = entityRegistry.getEntitySpec(entity); + AspectSpec keySpec = entitySpec.getKeyAspectSpec(); + List urns = new ArrayList<>(); + for (int i = 0; i < count; i++) { + Urn urn = UrnUtils.getUrn(String.format(urnTemplate, i)); + urns.add(urn); + when(entityService.getKeyAspectSpec(urn)).thenReturn(keySpec); + } + return urns; + } +} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java index ca31ba6e91b9db..ecac6ad05b0718 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java @@ -30,21 +30,11 @@ public class RestoreGlossaryIndicesTest { + private static final String VERSION_1 = "1"; + private static final String VERSION_2 = "2"; private static final String GLOSSARY_UPGRADE_URN = String.format("urn:li:%s:%s", Constants.DATA_HUB_UPGRADE_ENTITY_NAME, "restore-glossary-indices-ui"); - @Test - public void testExecuteFirstTime() throws Exception { - final Urn glossaryTermUrn = Urn.createFromString("urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"); - final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); - final EntityService mockService = Mockito.mock(EntityService.class); - final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); - final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); - - final Urn upgradeEntityUrn = Urn.createFromString(GLOSSARY_UPGRADE_URN); - Mockito.when(mockService.exists(upgradeEntityUrn)).thenReturn(false); - - - // Mock termInfoResponses and getting aspectSpec + private void mockGetTermInfo(Urn glossaryTermUrn, EntitySearchService mockSearchService, EntityService mockService) throws Exception { Map termInfoAspects = new HashMap<>(); termInfoAspects.put(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(new GlossaryTermInfo().setName("test").data()))); Map termInfoResponses = new HashMap<>(); @@ -52,12 +42,13 @@ public void testExecuteFirstTime() throws Exception { Mockito.when(mockSearchService.search(Constants.GLOSSARY_TERM_ENTITY_NAME, "", null, null, 0, 1000)) .thenReturn(new SearchResult().setNumEntities(1).setEntities(new SearchEntityArray(ImmutableList.of(new SearchEntity().setEntity(glossaryTermUrn))))); Mockito.when(mockService.getEntitiesV2( - Constants.GLOSSARY_TERM_ENTITY_NAME, + Constants.GLOSSARY_TERM_ENTITY_NAME, new HashSet<>(Collections.singleton(glossaryTermUrn)), Collections.singleton(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME))) .thenReturn(termInfoResponses); + } - // Mock nodeInfoResponses and getting aspectSpec + private void mockGetNodeInfo(Urn glossaryNodeUrn, EntitySearchService mockSearchService, EntityService mockService) throws Exception { Map nodeInfoAspects = new HashMap<>(); nodeInfoAspects.put(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(new GlossaryNodeInfo().setName("test").data()))); Map nodeInfoResponses = new HashMap<>(); @@ -65,12 +56,14 @@ public void testExecuteFirstTime() throws Exception { Mockito.when(mockSearchService.search(Constants.GLOSSARY_NODE_ENTITY_NAME, "", null, null, 0, 1000)) .thenReturn(new SearchResult().setNumEntities(1).setEntities(new SearchEntityArray(ImmutableList.of(new SearchEntity().setEntity(glossaryNodeUrn))))); Mockito.when(mockService.getEntitiesV2( - Constants.GLOSSARY_NODE_ENTITY_NAME, + Constants.GLOSSARY_NODE_ENTITY_NAME, new HashSet<>(Collections.singleton(glossaryNodeUrn)), Collections.singleton(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME) )) .thenReturn(nodeInfoResponses); + } + private AspectSpec mockGlossaryAspectSpecs(EntityRegistry mockRegistry) { EntitySpec entitySpec = Mockito.mock(EntitySpec.class); AspectSpec aspectSpec = Mockito.mock(AspectSpec.class); // Mock for Terms @@ -80,10 +73,95 @@ public void testExecuteFirstTime() throws Exception { Mockito.when(mockRegistry.getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME)).thenReturn(entitySpec); Mockito.when(entitySpec.getAspectSpec(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME)).thenReturn(aspectSpec); + return aspectSpec; + } + + @Test + public void testExecuteFirstTime() throws Exception { + final Urn glossaryTermUrn = Urn.createFromString("urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"); + final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); + final EntityService mockService = Mockito.mock(EntityService.class); + final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); + final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); + + final Urn upgradeEntityUrn = Urn.createFromString(GLOSSARY_UPGRADE_URN); + Mockito.when(mockService.getEntityV2( + Constants.DATA_HUB_UPGRADE_ENTITY_NAME, + upgradeEntityUrn, + Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME) + )).thenReturn(null); + + mockGetTermInfo(glossaryTermUrn, mockSearchService, mockService); + mockGetNodeInfo(glossaryNodeUrn, mockSearchService, mockService); + + AspectSpec aspectSpec = mockGlossaryAspectSpecs(mockRegistry); + + RestoreGlossaryIndices restoreIndicesStep = new RestoreGlossaryIndices(mockService, mockSearchService, mockRegistry); + restoreIndicesStep.execute(); + + + Mockito.verify(mockRegistry, Mockito.times(1)).getEntitySpec(Constants.GLOSSARY_TERM_ENTITY_NAME); + Mockito.verify(mockRegistry, Mockito.times(1)).getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME); + Mockito.verify(mockService, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class) + ); + Mockito.verify(mockService, Mockito.times(1)).produceMetadataChangeLog( + Mockito.eq(glossaryTermUrn), + Mockito.eq(Constants.GLOSSARY_TERM_ENTITY_NAME), + Mockito.eq(Constants.GLOSSARY_TERM_INFO_ASPECT_NAME), + Mockito.eq(aspectSpec), + Mockito.eq(null), + Mockito.any(), + Mockito.eq(null), + Mockito.eq(null), + Mockito.any(), + Mockito.eq(ChangeType.RESTATE) + ); + Mockito.verify(mockService, Mockito.times(1)).produceMetadataChangeLog( + Mockito.eq(glossaryNodeUrn), + Mockito.eq(Constants.GLOSSARY_NODE_ENTITY_NAME), + Mockito.eq(Constants.GLOSSARY_NODE_INFO_ASPECT_NAME), + Mockito.eq(aspectSpec), + Mockito.eq(null), + Mockito.any(), + Mockito.eq(null), + Mockito.eq(null), + Mockito.any(), + Mockito.eq(ChangeType.RESTATE) + ); + } + + @Test + public void testExecutesWithNewVersion() throws Exception { + final Urn glossaryTermUrn = Urn.createFromString("urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"); + final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); + final EntityService mockService = Mockito.mock(EntityService.class); + final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); + final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); + + final Urn upgradeEntityUrn = Urn.createFromString(GLOSSARY_UPGRADE_URN); + com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(VERSION_2); + Map upgradeRequestAspects = new HashMap<>(); + upgradeRequestAspects.put(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(upgradeRequest.data()))); + EntityResponse response = new EntityResponse().setAspects(new EnvelopedAspectMap(upgradeRequestAspects)); + Mockito.when(mockService.getEntityV2( + Constants.DATA_HUB_UPGRADE_ENTITY_NAME, + upgradeEntityUrn, + Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME) + )).thenReturn(response); + + mockGetTermInfo(glossaryTermUrn, mockSearchService, mockService); + mockGetNodeInfo(glossaryNodeUrn, mockSearchService, mockService); + + AspectSpec aspectSpec = mockGlossaryAspectSpecs(mockRegistry); + RestoreGlossaryIndices restoreIndicesStep = new RestoreGlossaryIndices(mockService, mockSearchService, mockRegistry); restoreIndicesStep.execute(); + Mockito.verify(mockRegistry, Mockito.times(1)).getEntitySpec(Constants.GLOSSARY_TERM_ENTITY_NAME); + Mockito.verify(mockRegistry, Mockito.times(1)).getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME); Mockito.verify(mockService, Mockito.times(2)).ingestProposal( Mockito.any(MetadataChangeProposal.class), Mockito.any(AuditStamp.class) @@ -120,10 +198,24 @@ public void testDoesNotRunWhenAlreadyExecuted() throws Exception { final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); final EntityService mockService = Mockito.mock(EntityService.class); final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); + final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); final Urn upgradeEntityUrn = Urn.createFromString(GLOSSARY_UPGRADE_URN); - Mockito.when(mockService.exists(upgradeEntityUrn)).thenReturn(true); + com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = new com.linkedin.upgrade.DataHubUpgradeRequest().setVersion(VERSION_1); + Map upgradeRequestAspects = new HashMap<>(); + upgradeRequestAspects.put(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(upgradeRequest.data()))); + EntityResponse response = new EntityResponse().setAspects(new EnvelopedAspectMap(upgradeRequestAspects)); + Mockito.when(mockService.getEntityV2( + Constants.DATA_HUB_UPGRADE_ENTITY_NAME, + upgradeEntityUrn, + Collections.singleton(Constants.DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME) + )).thenReturn(response); + + RestoreGlossaryIndices restoreIndicesStep = new RestoreGlossaryIndices(mockService, mockSearchService, mockRegistry); + restoreIndicesStep.execute(); + Mockito.verify(mockRegistry, Mockito.times(0)).getEntitySpec(Constants.GLOSSARY_TERM_ENTITY_NAME); + Mockito.verify(mockRegistry, Mockito.times(0)).getEntitySpec(Constants.GLOSSARY_NODE_ENTITY_NAME); Mockito.verify(mockSearchService, Mockito.times(0)).search(Constants.GLOSSARY_TERM_ENTITY_NAME, "", null, null, 0, 1000); Mockito.verify(mockSearchService, Mockito.times(0)).search(Constants.GLOSSARY_NODE_ENTITY_NAME, "", null, null, 0, 1000); Mockito.verify(mockService, Mockito.times(0)).ingestProposal( diff --git a/metadata-service/factories/src/test/resources/test-entity-registry.yaml b/metadata-service/factories/src/test/resources/test-entity-registry.yaml new file mode 100644 index 00000000000000..45aa9b9554fb4b --- /dev/null +++ b/metadata-service/factories/src/test/resources/test-entity-registry.yaml @@ -0,0 +1,10 @@ +id: test-registry +entities: + - name: corpuser + keyAspect: corpUserKey + aspects: + - corpUserInfo + - name: chart + keyAspect: chartKey + aspects: + - domains diff --git a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java index eb50f238277deb..49e2b501b85917 100644 --- a/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java +++ b/metadata-service/graphql-servlet-impl/src/main/java/com/datahub/graphql/GraphQLController.java @@ -1,17 +1,22 @@ package com.datahub.graphql; -import com.datahub.authentication.AuthenticationContext; +import com.codahale.metrics.MetricRegistry; import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; import com.datahub.authorization.AuthorizerChain; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.datahub.graphql.GraphQLEngine; +import com.linkedin.metadata.utils.metrics.MetricUtils; import graphql.ExecutionResult; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -104,6 +109,9 @@ CompletableFuture> postGraphQL(HttpEntity httpEnt * Format & Return Response */ try { + submitMetrics(executionResult); + // Remove tracing from response to reduce bulk, not used by the frontend + executionResult.getExtensions().remove("tracing"); String responseBodyStr = new ObjectMapper().writeValueAsString(executionResult.toSpecification()); return new ResponseEntity<>(responseBodyStr, HttpStatus.OK); } catch (IllegalArgumentException | JsonProcessingException e) { @@ -117,4 +125,24 @@ CompletableFuture> postGraphQL(HttpEntity httpEnt void getGraphQL(HttpServletRequest request, HttpServletResponse response) { throw new UnsupportedOperationException("GraphQL gets not supported."); } + + @SuppressWarnings("unchecked") + private void submitMetrics(ExecutionResult executionResult) { + try { + Object tracingInstrumentation = executionResult.getExtensions().get("tracing"); + if (tracingInstrumentation instanceof Map) { + Map tracingMap = (Map) tracingInstrumentation; + long totalDuration = TimeUnit.NANOSECONDS.toMillis((long) tracingMap.get("duration")); + Map executionData = (Map) tracingMap.get("execution"); + // Extract top level resolver, parent is top level query. Assumes single query per call. + List> resolvers = (List>) executionData.get("resolvers"); + Optional> + parentResolver = resolvers.stream().filter(resolver -> resolver.get("parentType").equals("Query")).findFirst(); + String fieldName = parentResolver.isPresent() ? (String) parentResolver.get().get("fieldName") : "UNKNOWN"; + MetricUtils.get().histogram(MetricRegistry.name(this.getClass(), fieldName)).update(totalDuration); + } + } catch (Exception e) { + log.error("Unable to submit metrics for GraphQL call.", e); + } + } } diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle index f5cdfe42cbff68..08f019bed70fb2 100644 --- a/metadata-service/openapi-servlet/build.gradle +++ b/metadata-service/openapi-servlet/build.gradle @@ -2,6 +2,7 @@ apply plugin: 'java' dependencies { + compile project(':metadata-service:auth-api') compile project(':metadata-service:factories') compile externalDependency.reflections diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java index fd73553e0e58cd..2209180bb47e6a 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/entities/EntitiesController.java @@ -2,6 +2,8 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -104,9 +106,12 @@ public ResponseEntity> postEntities( @RequestBody @Nonnull List aspectRequests) { log.info("INGEST PROPOSAL proposal: {}", aspectRequests); + Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); + List> responses = aspectRequests.stream() .map(MappingUtil::mapToProposal) - .map(proposal -> MappingUtil.ingestProposal(proposal, _entityService, _objectMapper)) + .map(proposal -> MappingUtil.ingestProposal(proposal, actorUrnStr, _entityService, _objectMapper)) .collect(Collectors.toList()); if (responses.stream().anyMatch(Pair::getSecond)) { return ResponseEntity.status(HttpStatus.CREATED) @@ -140,10 +145,14 @@ public ResponseEntity> deleteEntities( List deleteRequests = entityUrns.stream() .map(entityUrn -> MappingUtil.createStatusRemoval(entityUrn, _entityService)) .collect(Collectors.toList()); + + Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); + return ResponseEntity.ok(Collections.singletonList(RollbackRunResultDto.builder() .rowsRolledBack(deleteRequests.stream() .map(MappingUtil::mapToProposal) - .map(proposal -> MappingUtil.ingestProposal(proposal, _entityService, _objectMapper)) + .map(proposal -> MappingUtil.ingestProposal(proposal, actorUrnStr, _entityService, _objectMapper)) .filter(Pair::getSecond) .map(Pair::getFirst) .map(urnString -> new AspectRowSummary().urn(urnString)) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java index 0d7ed888c8f8b9..5f968a56ee215f 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/platform/entities/PlatformEntitiesController.java @@ -1,5 +1,7 @@ package io.datahubproject.openapi.platform.entities; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.entity.EntityService; import com.linkedin.util.Pair; @@ -44,8 +46,11 @@ public ResponseEntity> postEntities( @RequestBody @Nonnull List metadataChangeProposals) { log.info("INGEST PROPOSAL proposal: {}", metadataChangeProposals); + Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); + List> responses = metadataChangeProposals.stream() - .map(proposal -> MappingUtil.ingestProposal(proposal, _entityService, _objectMapper)) + .map(proposal -> MappingUtil.ingestProposal(proposal, actorUrnStr, _entityService, _objectMapper)) .collect(Collectors.toList()); if (responses.stream().anyMatch(Pair::getSecond)) { return ResponseEntity.status(HttpStatus.CREATED) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java index 8d071965353229..15a90afa3612b7 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java @@ -14,7 +14,6 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.Aspect; import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.RollbackRunResult; import com.linkedin.metadata.entity.ValidationException; @@ -246,13 +245,13 @@ public static GenericAspect convertGenericAspect(@Nonnull io.datahubproject.open } } - public static Pair ingestProposal(MetadataChangeProposal metadataChangeProposal, EntityService entityService, + public static Pair ingestProposal(MetadataChangeProposal metadataChangeProposal, String actorUrn, EntityService entityService, ObjectMapper objectMapper) { // TODO: Use the actor present in the IC. Timer.Context context = MetricUtils.timer("postEntity").time(); final com.linkedin.common.AuditStamp auditStamp = new com.linkedin.common.AuditStamp().setTime(System.currentTimeMillis()) - .setActor(UrnUtils.getUrn(Constants.UNKNOWN_ACTOR)); + .setActor(UrnUtils.getUrn(actorUrn)); io.datahubproject.openapi.generated.KafkaAuditHeader auditHeader = metadataChangeProposal.getAuditHeader(); com.linkedin.mxe.MetadataChangeProposal serviceProposal = diff --git a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java index d81069f07efd6a..b158c4ca6ad00e 100644 --- a/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java +++ b/metadata-service/openapi-servlet/src/test/java/entities/EntitiesControllerTest.java @@ -1,5 +1,9 @@ package entities; +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.event.EventProducer; @@ -34,6 +38,7 @@ import org.testng.annotations.Test; import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.when; public class EntitiesControllerTest { @@ -53,6 +58,9 @@ public void setup() EventProducer mockEntityEventProducer = Mockito.mock(EventProducer.class); MockEntityService mockEntityService = new MockEntityService(aspectDao, mockEntityEventProducer, mockEntityRegistry); _entitiesController = new EntitiesController(mockEntityService, new ObjectMapper()); + Authentication authentication = Mockito.mock(Authentication.class); + when(authentication.getActor()).thenReturn(new Actor(ActorType.USER, "datahub")); + AuthenticationContext.setAuthentication(authentication); } EntitiesController _entitiesController; diff --git a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entities.restspec.json b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entities.restspec.json index 137f6e431dd696..9116dd03c8733b 100644 --- a/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entities.restspec.json +++ b/metadata-service/restli-api/src/main/idl/com.linkedin.entity.entities.restspec.json @@ -114,6 +114,13 @@ "optional" : true } ], "returns" : "com.linkedin.metadata.run.DeleteReferencesResponse" + }, { + "name" : "exists", + "parameters" : [ { + "name" : "urn", + "type" : "string" + } ], + "returns" : "boolean" }, { "name" : "filter", "parameters" : [ { @@ -257,6 +264,10 @@ "name" : "input", "type" : "string", "optional" : true + }, { + "name" : "maxHops", + "type" : "int", + "optional" : true }, { "name" : "filter", "type" : "com.linkedin.metadata.query.filter.Filter", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json index e2b51e8d62d54a..0f54e05ccd5e4b 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.analytics.analytics.snapshot.json @@ -56,7 +56,7 @@ "type" : "enum", "name" : "Condition", "doc" : "The matching condition in a filter criterion", - "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ], + "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IS_NULL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ], "symbolDocs" : { "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile", "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event", @@ -64,6 +64,7 @@ "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5", "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5", "IN" : "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]", + "IS_NULL" : "Represent the relation: field is null, e.g. platform is null", "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3", "LESS_THAN_OR_EQUAL_TO" : "Represent the relation less than or equal to, e.g. ownerCount <= 3", "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView" diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index bf7b862354e874..ec7f8562db9648 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -146,7 +146,7 @@ "type" : "enum", "name" : "Condition", "doc" : "The matching condition in a filter criterion", - "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ], + "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IS_NULL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ], "symbolDocs" : { "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile", "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event", @@ -154,6 +154,7 @@ "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5", "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5", "IN" : "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]", + "IS_NULL" : "Represent the relation: field is null, e.g. platform is null", "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3", "LESS_THAN_OR_EQUAL_TO" : "Represent the relation less than or equal to, e.g. ownerCount <= 3", "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView" @@ -1200,6 +1201,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 67ccf555672907..aacd347b14c86f 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -1217,6 +1217,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", @@ -5147,7 +5162,7 @@ "name" : "Condition", "namespace" : "com.linkedin.metadata.query.filter", "doc" : "The matching condition in a filter criterion", - "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ], + "symbols" : [ "CONTAIN", "END_WITH", "EQUAL", "IS_NULL", "GREATER_THAN", "GREATER_THAN_OR_EQUAL_TO", "IN", "LESS_THAN", "LESS_THAN_OR_EQUAL_TO", "START_WITH" ], "symbolDocs" : { "CONTAIN" : "Represent the relation: String field contains value, e.g. name contains Profile", "END_WITH" : "Represent the relation: String field ends with value, e.g. name ends with Event", @@ -5155,6 +5170,7 @@ "GREATER_THAN" : "Represent the relation greater than, e.g. ownerCount > 5", "GREATER_THAN_OR_EQUAL_TO" : "Represent the relation greater than or equal to, e.g. ownerCount >= 5", "IN" : "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]", + "IS_NULL" : "Represent the relation: field is null, e.g. platform is null", "LESS_THAN" : "Represent the relation less than, e.g. ownerCount < 3", "LESS_THAN_OR_EQUAL_TO" : "Represent the relation less than or equal to, e.g. ownerCount <= 3", "START_WITH" : "Represent the relation: String field starts with value, e.g. name starts with PageView" @@ -5684,6 +5700,13 @@ "optional" : true } ], "returns" : "com.linkedin.metadata.run.DeleteReferencesResponse" + }, { + "name" : "exists", + "parameters" : [ { + "name" : "urn", + "type" : "string" + } ], + "returns" : "boolean" }, { "name" : "filter", "parameters" : [ { @@ -5827,6 +5850,10 @@ "name" : "input", "type" : "string", "optional" : true + }, { + "name" : "maxHops", + "type" : "int", + "optional" : true }, { "name" : "filter", "type" : "com.linkedin.metadata.query.filter.Filter", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json index c428918748e25a..73c355a23f7476 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json @@ -93,12 +93,53 @@ "name" : "created", "type" : "com.linkedin.common.AuditStamp", "doc" : "The audit stamp detailing who the aspect was created by and when\n" + }, { + "name" : "systemMetadata", + "type" : { + "type" : "record", + "name" : "SystemMetadata", + "namespace" : "com.linkedin.mxe", + "doc" : "Metadata associated with each metadata change that is processed by the system", + "fields" : [ { + "name" : "lastObserved", + "type" : "long", + "doc" : "The timestamp the metadata was observed at", + "default" : 0, + "optional" : true + }, { + "name" : "runId", + "type" : "string", + "doc" : "The run id that produced the metadata. Populated in case of batch-ingestion.", + "default" : "no-run-id-provided", + "optional" : true + }, { + "name" : "registryName", + "type" : "string", + "doc" : "The model registry name that was used to process this event", + "optional" : true + }, { + "name" : "registryVersion", + "type" : "string", + "doc" : "The model registry version that was used to process this event", + "optional" : true + }, { + "name" : "properties", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "Additional properties", + "optional" : true + } ] + }, + "doc" : "The system metadata for this aspect\n", + "optional" : true } ] } }, "doc" : "A map of aspect name to aspect\n" } ] - }, "com.linkedin.entity.EnvelopedAspect" ], + }, "com.linkedin.entity.EnvelopedAspect", "com.linkedin.mxe.SystemMetadata" ], "schema" : { "name" : "entitiesV2", "namespace" : "com.linkedin.entity", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json index 6e187fba118560..d9f36ddba40267 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json @@ -102,12 +102,53 @@ "name" : "created", "type" : "com.linkedin.common.AuditStamp", "doc" : "The audit stamp detailing who the aspect was created by and when\n" + }, { + "name" : "systemMetadata", + "type" : { + "type" : "record", + "name" : "SystemMetadata", + "namespace" : "com.linkedin.mxe", + "doc" : "Metadata associated with each metadata change that is processed by the system", + "fields" : [ { + "name" : "lastObserved", + "type" : "long", + "doc" : "The timestamp the metadata was observed at", + "default" : 0, + "optional" : true + }, { + "name" : "runId", + "type" : "string", + "doc" : "The run id that produced the metadata. Populated in case of batch-ingestion.", + "default" : "no-run-id-provided", + "optional" : true + }, { + "name" : "registryName", + "type" : "string", + "doc" : "The model registry name that was used to process this event", + "optional" : true + }, { + "name" : "registryVersion", + "type" : "string", + "doc" : "The model registry version that was used to process this event", + "optional" : true + }, { + "name" : "properties", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "Additional properties", + "optional" : true + } ] + }, + "doc" : "The system metadata for this aspect\n", + "optional" : true } ] } }, "doc" : "A map of aspect name to aspect\n" } ] - }, "com.linkedin.entity.EnvelopedAspect" ], + }, "com.linkedin.entity.EnvelopedAspect", "com.linkedin.mxe.SystemMetadata" ], "schema" : { "name" : "entitiesVersionedV2", "namespace" : "com.linkedin.entity", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index 8bdf251767c4c1..4412438fe28125 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -960,6 +960,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index cf56b50965cd0e..9174ef8f9b455b 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -1217,6 +1217,21 @@ "name" : "Contains" } } + }, { + "name" : "datasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Datasets consumed by a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "isLineage" : true, + "name" : "Consumes" + } + } }, { "name" : "lastModified", "type" : "com.linkedin.common.ChangeAuditStamps", diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java index 3303e095f44b11..6bdf8cafa65e66 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -187,6 +187,7 @@ public SearchResult searchAcrossEntities(@Nonnull List entities, @Nonnul * @param direction Direction of the relationship * @param entities list of entities to search (If empty, searches across all entities) * @param input the search input text + * @param maxHops the max number of hops away to search for. If null, searches all hops. * @param filter the request map with fields and values as filters to be applied to search hits * @param sortCriterion {@link SortCriterion} to be applied to search results * @param start index to start the search from @@ -195,7 +196,7 @@ public SearchResult searchAcrossEntities(@Nonnull List entities, @Nonnul */ @Nonnull public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull LineageDirection direction, - @Nonnull List entities, @Nonnull String input, @Nullable Filter filter, + @Nonnull List entities, @Nonnull String input, @Nullable Integer maxHops, @Nullable Filter filter, @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException; @@ -228,6 +229,12 @@ public ListUrnsResult listUrns(@Nonnull final String entityName, final int start public void deleteEntity(@Nonnull final Urn urn, @Nonnull final Authentication authentication) throws RemoteInvocationException; + /** + * Delete all references to an entity with a particular urn. + */ + public void deleteEntityReferences(@Nonnull final Urn urn, @Nonnull final Authentication authentication) + throws RemoteInvocationException; + /** * Filters entities based on a particular Filter and Sort criterion * @@ -243,6 +250,17 @@ public void deleteEntity(@Nonnull final Urn urn, @Nonnull final Authentication a public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull Authentication authentication) throws RemoteInvocationException; + /** + * Checks whether an entity with a given urn exists + * + * @param urn the urn of the entity + * @return true if an entity exists, i.e. there are > 0 aspects in the DB for the entity. This means that the entity + * has not been hard-deleted. + * @throws RemoteInvocationException + */ + @Nonnull + public boolean exists(@Nonnull Urn urn, @Nonnull Authentication authentication) throws RemoteInvocationException; + @Nullable @Deprecated public VersionedAspect getAspect(@Nonnull String urn, @Nonnull String aspect, @Nonnull Long version, diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java index dca2fc1b191a60..f7913f0feaf092 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java @@ -18,6 +18,7 @@ import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.entity.DeleteEntityService; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.graph.LineageDirection; @@ -33,6 +34,7 @@ import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.client.CachingEntitySearchService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.PlatformEvent; @@ -63,11 +65,13 @@ public class JavaEntityClient implements EntityClient { private final Clock _clock = Clock.systemUTC(); private final EntityService _entityService; - private final EventProducer _eventProducer; + private final DeleteEntityService _deleteEntityService; private final EntitySearchService _entitySearchService; + private final CachingEntitySearchService _cachingEntitySearchService; private final SearchService _searchService; - private final TimeseriesAspectService _timeseriesAspectService; private final LineageSearchService _lineageSearchService; + private final TimeseriesAspectService _timeseriesAspectService; + private final EventProducer _eventProducer; @Nullable public EntityResponse getV2( @@ -117,10 +121,11 @@ public Map batchGet(@Nonnull final Set urns, @Nonnull final Au } /** - * Gets browse snapshot of a given path + * Gets autocomplete results * + * @param entityType the type of entity to autocomplete against * @param query search query - * @param field field of the dataset + * @param field field of the dataset to autocomplete against * @param requestFilters autocomplete filters * @param limit max number of autocomplete results * @throws RemoteInvocationException @@ -133,12 +138,13 @@ public AutoCompleteResult autoComplete( @Nonnull int limit, @Nullable String field, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return _entitySearchService.autoComplete(entityType, query, field, newFilter(requestFilters), limit); + return _cachingEntitySearchService.autoComplete(entityType, query, field, newFilter(requestFilters), limit, null); } - /**; + /** * Gets autocomplete results * + * @param entityType the type of entity to autocomplete against * @param query search query * @param requestFilters autocomplete filters * @param limit max number of autocomplete results @@ -151,7 +157,7 @@ public AutoCompleteResult autoComplete( @Nonnull Map requestFilters, @Nonnull int limit, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return _entitySearchService.autoComplete(entityType, query, "", newFilter(requestFilters), limit); + return _cachingEntitySearchService.autoComplete(entityType, query, "", newFilter(requestFilters), limit, null); } /** @@ -176,6 +182,7 @@ public BrowseResult browse( } @SneakyThrows + @Deprecated public void update(@Nonnull final Entity entity, @Nonnull final Authentication authentication) throws RemoteInvocationException { Objects.requireNonNull(authentication, "authentication must not be null"); @@ -186,6 +193,7 @@ public void update(@Nonnull final Entity entity, @Nonnull final Authentication a } @SneakyThrows + @Deprecated public void updateWithSystemMetadata( @Nonnull final Entity entity, @Nullable final SystemMetadata systemMetadata, @@ -200,16 +208,17 @@ public void updateWithSystemMetadata( auditStamp.setTime(Clock.systemUTC().millis()); _entityService.ingestEntity(entity, auditStamp, systemMetadata); + tryIndexRunId(com.datahub.util.ModelUtils.getUrnFromSnapshotUnion(entity.getValue()), systemMetadata); } @SneakyThrows + @Deprecated public void batchUpdate(@Nonnull final Set entities, @Nonnull final Authentication authentication) throws RemoteInvocationException { AuditStamp auditStamp = new AuditStamp(); auditStamp.setActor(Urn.createFromString(authentication.getActor().toUrnStr())); auditStamp.setTime(Clock.systemUTC().millis()); - - _entityService.ingestEntities(entities.stream().collect(Collectors.toList()), auditStamp, ImmutableList.of()); + _entityService.ingestEntities(entities.stream().collect(Collectors.toList()), auditStamp, ImmutableList.of()); } /** @@ -236,6 +245,8 @@ public SearchResult search( } /** + * Deprecated! Use 'filter' or 'search' instead. + * * Filters for entities matching to a given query and filters * * @param requestFilters search filters @@ -244,6 +255,7 @@ public SearchResult search( * @return a set of list results * @throws RemoteInvocationException */ + @Deprecated @Nonnull public ListResult list( @Nonnull String entity, @@ -305,10 +317,10 @@ public SearchResult searchAcrossEntities( @Nonnull @Override public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull LineageDirection direction, - @Nonnull List entities, @Nullable String input, @Nullable Filter filter, + @Nonnull List entities, @Nullable String input, @Nullable Integer maxHops, @Nullable Filter filter, @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { - return _lineageSearchService.searchAcrossLineage(sourceUrn, direction, entities, input, filter, + return _lineageSearchService.searchAcrossLineage(sourceUrn, direction, entities, input, maxHops, filter, sortCriterion, start, count); } @@ -350,6 +362,12 @@ public void deleteEntity(@Nonnull final Urn urn, @Nonnull final Authentication a _entityService.deleteUrn(urn); } + @Override + public void deleteEntityReferences(@Nonnull Urn urn, @Nonnull Authentication authentication) + throws RemoteInvocationException { + _deleteEntityService.deleteReferencesTo(urn, false); + } + @Nonnull @Override public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Nullable SortCriterion sortCriterion, @@ -357,6 +375,12 @@ public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Null return _entitySearchService.filter(entity, filter, sortCriterion, start, count); } + @Nonnull + @Override + public boolean exists(@Nonnull Urn urn, @Nonnull final Authentication authentication) throws RemoteInvocationException { + return _entityService.exists(urn); + } + @SneakyThrows @Override public VersionedAspect getAspect(@Nonnull String urn, @Nonnull String aspect, @Nonnull Long version, @@ -413,6 +437,7 @@ public String ingestProposal(@Nonnull MetadataChangeProposal metadataChangePropo Urn urn = _entityService.ingestProposal(metadataChangeProposal, auditStamp).getUrn(); additionalChanges.forEach(proposal -> _entityService.ingestProposal(proposal, auditStamp)); + tryIndexRunId(urn, metadataChangeProposal.getSystemMetadata()); return urn.toString(); } @@ -455,4 +480,10 @@ public void producePlatformEvent( @Nonnull Authentication authentication) throws Exception { _eventProducer.producePlatformEvent(name, key, event); } + + private void tryIndexRunId(Urn entityUrn, @Nullable SystemMetadata systemMetadata) { + if (systemMetadata != null && systemMetadata.hasRunId()) { + _entitySearchService.appendRunId(entityUrn.getEntityType(), entityUrn, systemMetadata.getRunId()); + } + } } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index 27defaf417e0e1..0de87f0620c61e 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -18,7 +18,9 @@ import com.linkedin.entity.EntitiesDoBatchGetTotalEntityCountRequestBuilder; import com.linkedin.entity.EntitiesDoBatchIngestRequestBuilder; import com.linkedin.entity.EntitiesDoBrowseRequestBuilder; +import com.linkedin.entity.EntitiesDoDeleteReferencesRequestBuilder; import com.linkedin.entity.EntitiesDoDeleteRequestBuilder; +import com.linkedin.entity.EntitiesDoExistsRequestBuilder; import com.linkedin.entity.EntitiesDoFilterRequestBuilder; import com.linkedin.entity.EntitiesDoGetBrowsePathsRequestBuilder; import com.linkedin.entity.EntitiesDoIngestRequestBuilder; @@ -416,7 +418,7 @@ public SearchResult searchAcrossEntities(@Nonnull List entities, @Nonnul @Nonnull @Override public LineageSearchResult searchAcrossLineage(@Nonnull Urn sourceUrn, @Nonnull LineageDirection direction, - @Nonnull List entities, @Nonnull String input, @Nullable Filter filter, + @Nonnull List entities, @Nonnull String input, @Nullable Integer maxHops, @Nullable Filter filter, @Nullable SortCriterion sortCriterion, int start, int count, @Nonnull final Authentication authentication) throws RemoteInvocationException { @@ -487,6 +489,16 @@ public void deleteEntity(@Nonnull final Urn urn, @Nonnull final Authentication a sendClientRequest(requestBuilder, authentication); } + /** + * Delete all references to a particular entity. + */ + @Override + public void deleteEntityReferences(@Nonnull Urn urn, @Nonnull Authentication authentication) + throws RemoteInvocationException { + EntitiesDoDeleteReferencesRequestBuilder requestBuilder = ENTITIES_REQUEST_BUILDERS.actionDeleteReferences().urnParam(urn.toString()); + sendClientRequest(requestBuilder, authentication); + } + @Nonnull @Override public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Nullable SortCriterion sortCriterion, @@ -502,6 +514,14 @@ public SearchResult filter(@Nonnull String entity, @Nonnull Filter filter, @Null return sendClientRequest(requestBuilder, authentication).getEntity(); } + @Nonnull + @Override + public boolean exists(@Nonnull Urn urn, @Nonnull final Authentication authentication) throws RemoteInvocationException { + EntitiesDoExistsRequestBuilder requestBuilder = ENTITIES_REQUEST_BUILDERS.actionExists() + .urnParam(urn.toString()); + return sendClientRequest(requestBuilder, authentication).getEntity(); + } + /** * Gets aspect at version for an entity * diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 2d90b706eca1ef..1e6537da6234a1 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -12,6 +12,7 @@ import com.linkedin.metadata.entity.ValidationException; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.restli.RestliUtil; +import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.parseq.Task; @@ -35,6 +36,7 @@ import javax.inject.Named; import lombok.extern.slf4j.Slf4j; +import static com.linkedin.metadata.resources.entity.ResourceUtils.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*; @@ -61,6 +63,10 @@ public class AspectResource extends CollectionResourceTaskTemplate ingestProposal( Authentication authentication = AuthenticationContext.getAuthentication(); String actorUrnStr = authentication.getActor().toUrnStr(); - // Getting actor from AuthenticationContext - log.debug(String.format("Retrieving AuthenticationContext for Actor with : %s", actorUrnStr)); - final AuditStamp auditStamp = - new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); + final AuditStamp auditStamp = new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); final List additionalChanges = AspectUtils.getAdditionalChanges(metadataChangeProposal, _entityService); @@ -140,11 +143,11 @@ public Task ingestProposal( try { Urn urn = _entityService.ingestProposal(metadataChangeProposal, auditStamp).getUrn(); additionalChanges.forEach(proposal -> _entityService.ingestProposal(proposal, auditStamp)); + tryIndexRunId(urn, metadataChangeProposal.getSystemMetadata(), _entitySearchService); return urn.toString(); } catch (ValidationException e) { throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, e.getMessage()); } }, MetricRegistry.name(this.getClass(), "ingestProposal")); } - } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index bb372df5ae0320..c5dc43a8ee6e1f 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -67,6 +67,7 @@ import org.apache.maven.artifact.versioning.ComparableVersion; import static com.linkedin.metadata.entity.ValidationUtils.*; +import static com.linkedin.metadata.resources.entity.ResourceUtils.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*; import static com.linkedin.metadata.utils.PegasusUtils.*; @@ -85,10 +86,13 @@ public class EntityResource extends CollectionResourceTaskTemplate ingest(@ActionParam(PARAM_ENTITY) @Nonnull Entity entity, Authentication authentication = AuthenticationContext.getAuthentication(); String actorUrnStr = authentication.getActor().toUrnStr(); - // Getting actor from AuthenticationContext - log.debug(String.format("Retrieving AuthenticationContext for Actor with : %s", actorUrnStr)); final AuditStamp auditStamp = new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); // variables referenced in lambdas are required to be final final SystemMetadata finalSystemMetadata = systemMetadata; return RestliUtil.toTask(() -> { _entityService.ingestEntity(entity, auditStamp, finalSystemMetadata); + tryIndexRunId(com.datahub.util.ModelUtils.getUrnFromSnapshotUnion(entity.getValue()), systemMetadata, _entitySearchService); return null; }, MetricRegistry.name(this.getClass(), "ingest")); } @@ -224,8 +227,6 @@ public Task batchIngest(@ActionParam(PARAM_ENTITIES) @Nonnull Entity[] ent Authentication authentication = AuthenticationContext.getAuthentication(); String actorUrnStr = authentication.getActor().toUrnStr(); - // Getting actor from AuthenticationContext - log.debug(String.format("Retrieving AuthenticationContext for Actor with : %s", actorUrnStr)); final AuditStamp auditStamp = new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); if (systemMetadataList == null) { @@ -240,8 +241,14 @@ public Task batchIngest(@ActionParam(PARAM_ENTITIES) @Nonnull Entity[] ent .map(systemMetadata -> populateDefaultFieldsIfEmpty(systemMetadata)) .collect(Collectors.toList()); + SystemMetadata[] finalSystemMetadataList1 = systemMetadataList; return RestliUtil.toTask(() -> { _entityService.ingestEntities(Arrays.asList(entities), auditStamp, finalSystemMetadataList); + for (int i = 0; i < entities.length; i++) { + SystemMetadata systemMetadata = finalSystemMetadataList1[i]; + Entity entity = entities[i]; + tryIndexRunId(com.datahub.util.ModelUtils.getUrnFromSnapshotUnion(entity.getValue()), systemMetadata, _entitySearchService); + } return null; }, MetricRegistry.name(this.getClass(), "batchIngest")); } @@ -280,16 +287,16 @@ public Task searchAcrossEntities(@ActionParam(PARAM_ENTITIES) @Opt public Task searchAcrossLineage(@ActionParam(PARAM_URN) @Nonnull String urnStr, @ActionParam(PARAM_DIRECTION) String direction, @ActionParam(PARAM_ENTITIES) @Optional @Nullable String[] entities, - @ActionParam(PARAM_INPUT) @Optional @Nullable String input, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, - @ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion, @ActionParam(PARAM_START) int start, - @ActionParam(PARAM_COUNT) int count) throws URISyntaxException { + @ActionParam(PARAM_INPUT) @Optional @Nullable String input, @ActionParam(PARAM_MAX_HOPS) @Optional @Nullable Integer maxHops, + @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion, + @ActionParam(PARAM_START) int start, @ActionParam(PARAM_COUNT) int count) throws URISyntaxException { Urn urn = Urn.createFromString(urnStr); List entityList = entities == null ? Collections.emptyList() : Arrays.asList(entities); log.info("GET SEARCH RESULTS ACROSS RELATIONSHIPS for source urn {}, direction {}, entities {} with query {}", urnStr, direction, entityList, input); return RestliUtil.toTask( () -> _lineageSearchService.searchAcrossLineage(urn, LineageDirection.valueOf(direction), entityList, - input, filter, sortCriterion, start, count), "searchAcrossRelationships"); + input, maxHops, filter, sortCriterion, start, count), "searchAcrossRelationships"); } @Action(name = ACTION_LIST) @@ -350,7 +357,7 @@ private String stringifyRowCount(int size) { } /* - Used to delete all data related to a filter criteria based on registryId, runId etc. + Used to delete all data related to a filter criteria based on registryId, runId etc. */ @Action(name = "deleteAll") @Nonnull @@ -489,4 +496,14 @@ public Task filter(@ActionParam(PARAM_ENTITY) @Nonnull String enti return RestliUtil.toTask(() -> _entitySearchService.filter(entityName, filter, sortCriterion, start, count), MetricRegistry.name(this.getClass(), "search")); } + + @Action(name = ACTION_EXISTS) + @Nonnull + @WithSpan + public Task exists(@ActionParam(PARAM_URN) @Nonnull String urnStr) throws URISyntaxException { + log.info("EXISTS for {}", urnStr); + Urn urn = Urn.createFromString(urnStr); + return RestliUtil.toTask(() -> _entityService.exists(urn), + MetricRegistry.name(this.getClass(), "exists")); + } } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java index 679cb421bc5fe7..5918f453c6d880 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/ResourceUtils.java @@ -1,7 +1,11 @@ package com.linkedin.metadata.resources.entity; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.mxe.SystemMetadata; import java.util.Set; +import javax.annotation.Nullable; public class ResourceUtils { @@ -12,4 +16,13 @@ private ResourceUtils() { public static Set getAllAspectNames(final EntityService entityService, final String entityName) { return entityService.getEntityAspectNames(entityName); } + + public static void tryIndexRunId( + final Urn urn, + final @Nullable SystemMetadata systemMetadata, + final EntitySearchService entitySearchService) { + if (systemMetadata != null && systemMetadata.hasRunId()) { + entitySearchService.appendRunId(urn.getEntityType(), urn, systemMetadata.getRunId()); + } + } } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java index 1100d5a1949285..d2454e7bc23309 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/restli/RestliConstants.java @@ -19,6 +19,7 @@ private RestliConstants() { } public static final String ACTION_LIST_URNS_FROM_INDEX = "listUrnsFromIndex"; public static final String PARAM_INPUT = "input"; + public static final String PARAM_MAX_HOPS = "maxHops"; public static final String PARAM_ASPECTS = "aspects"; public static final String PARAM_FILTER = "filter"; public static final String PARAM_GROUP = "group"; diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 22dd867f23d7a3..2825f05e694072 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -486,6 +486,16 @@ "logoUrl": "/assets/platforms/pulsarlogo.png" } }, + { + "urn": "urn:li:dataPlatform:salesforce", + "aspect": { + "datasetNameDelimiter": ".", + "name": "salesforce", + "displayName": "Salesforce", + "type": "OTHERS", + "logoUrl": "/assets/platforms/logo-salesforce.svg" + } + }, { "urn": "urn:li:dataPlatform:unknown", "aspect": { @@ -494,5 +504,15 @@ "displayName": "N/A", "type": "OTHERS" } + }, + { + "urn": "urn:li:dataPlatform:delta-lake", + "aspect": { + "datasetNameDelimiter": ".", + "name": "delta-lake", + "displayName": "Delta Lake", + "type": "OTHERS", + "logoUrl": "/assets/platforms/deltalakelogo.png" + } } ] diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index 9a1596b48f581a..7a5b0781aae86c 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -20,7 +20,9 @@ "MANAGE_ACCESS_TOKENS", "MANAGE_DOMAINS", "MANAGE_TESTS", - "MANAGE_GLOSSARIES" + "MANAGE_GLOSSARIES", + "MANAGE_USER_CREDENTIALS", + "MANAGE_TAGS" ], "displayName":"Root User - All Platform Privileges", "description":"Grants full platform privileges to root datahub super user.", @@ -72,7 +74,8 @@ "actors":{ "resourceOwners":false, "allUsers":true, - "allGroups":false + "allGroups":false, + "users":[] }, "privileges":[ "MANAGE_POLICIES", @@ -83,7 +86,8 @@ "GENERATE_PERSONAL_ACCESS_TOKENS", "MANAGE_DOMAINS", "MANAGE_TESTS", - "MANAGE_GLOSSARIES" + "MANAGE_GLOSSARIES", + "MANAGE_TAGS" ], "displayName":"All Users - All Platform Privileges", "description":"Grants full platform privileges to ALL users of DataHub. Change this policy to alter that behavior.", diff --git a/metadata-service/war/src/main/webapp/WEB-INF/web.xml b/metadata-service/war/src/main/webapp/WEB-INF/web.xml index 4252067c2bdeff..9275ca87f22f38 100644 --- a/metadata-service/war/src/main/webapp/WEB-INF/web.xml +++ b/metadata-service/war/src/main/webapp/WEB-INF/web.xml @@ -23,7 +23,7 @@ org.springframework.web.context.ContextLoaderListener - com.linkedin.metadata.boot.BootstrapManagerApplicationListener + com.linkedin.metadata.boot.OnBootApplicationListener diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java index fc02f39c0949ab..ff14fcb5adfd16 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -15,6 +15,8 @@ public class Constants { public static final String DEFAULT_RUN_ID = "no-run-id-provided"; + public static final String GLOBAL_INVITE_TOKEN = "urn:li:inviteToken:global"; + /** * Entities */ @@ -45,6 +47,8 @@ public class Constants { public static final String DATA_PLATFORM_INSTANCE_ENTITY_NAME = "dataPlatformInstance"; public static final String ACCESS_TOKEN_ENTITY_NAME = "dataHubAccessToken"; public static final String DATA_HUB_UPGRADE_ENTITY_NAME = "dataHubUpgrade"; + public static final String INVITE_TOKEN_ENTITY_NAME = "inviteToken"; + /** * Aspects @@ -60,14 +64,18 @@ public class Constants { public static final String SUB_TYPES_ASPECT_NAME = "subTypes"; public static final String DEPRECATION_ASPECT_NAME = "deprecation"; public static final String OPERATION_ASPECT_NAME = "operation"; + public static final String SIBLINGS_ASPECT_NAME = "siblings"; + public static final String ORIGIN_ASPECT_NAME = "origin"; // User public static final String CORP_USER_KEY_ASPECT_NAME = "corpUserKey"; public static final String CORP_USER_EDITABLE_INFO_NAME = "corpUserEditableInfo"; public static final String GROUP_MEMBERSHIP_ASPECT_NAME = "groupMembership"; + public static final String NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME = "nativeGroupMembership"; public static final String CORP_USER_EDITABLE_INFO_ASPECT_NAME = "corpUserEditableInfo"; public static final String CORP_USER_INFO_ASPECT_NAME = "corpUserInfo"; public static final String CORP_USER_STATUS_ASPECT_NAME = "corpUserStatus"; + public static final String CORP_USER_CREDENTIALS_ASPECT_NAME = "corpUserCredentials"; // Group public static final String CORP_GROUP_KEY_ASPECT_NAME = "corpGroupKey"; @@ -91,11 +99,13 @@ public class Constants { public static final String CHART_INFO_ASPECT_NAME = "chartInfo"; public static final String EDITABLE_CHART_PROPERTIES_ASPECT_NAME = "editableChartProperties"; public static final String CHART_QUERY_ASPECT_NAME = "chartQuery"; + public static final String CHART_USAGE_STATISTICS_ASPECT_NAME = "chartUsageStatistics"; // Dashboard public static final String DASHBOARD_KEY_ASPECT_NAME = "dashboardKey"; public static final String DASHBOARD_INFO_ASPECT_NAME = "dashboardInfo"; public static final String EDITABLE_DASHBOARD_PROPERTIES_ASPECT_NAME = "editableDashboardProperties"; + public static final String DASHBOARD_USAGE_STATISTICS_ASPECT_NAME = "dashboardUsageStatistics"; // Notebook public static final String NOTEBOOK_KEY_ASPECT_NAME = "notebookKey"; @@ -217,6 +227,14 @@ public class Constants { public static final String DATA_HUB_UPGRADE_RESULT_ASPECT_NAME = "dataHubUpgradeResult"; + // Invite Token + public static final String INVITE_TOKEN_ASPECT_NAME = "inviteToken"; + + // Relationships + public static final String IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME = "IsMemberOfGroup"; + public static final String IS_MEMBER_OF_NATIVE_GROUP_RELATIONSHIP_NAME = "IsMemberOfNativeGroup"; + + // acryl-main only public static final String CHANGE_EVENT_PLATFORM_EVENT_NAME = "entityChangeEvent"; /** diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index 70e4ff12f55f32..16527a7b6cdac2 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -52,7 +52,6 @@ public class PoliciesConfig { "Generate Personal Access Tokens", "Generate personal access tokens for use with DataHub APIs."); - public static final Privilege MANAGE_ACCESS_TOKENS = Privilege.of( "MANAGE_ACCESS_TOKENS", "Manage All Access Tokens", @@ -75,6 +74,25 @@ public class PoliciesConfig { "Manage Glossaries", "Create, edit, and remove Glossary Entities"); + public static final Privilege MANAGE_USER_CREDENTIALS_PRIVILEGE = + Privilege.of("MANAGE_USER_CREDENTIALS", "Manage User Credentials", + "Manage credentials for native DataHub users, including inviting new users and resetting passwords"); + + public static final Privilege MANAGE_TAGS_PRIVILEGE = Privilege.of( + "MANAGE_TAGS", + "Manage Tags", + "Create and remove Tags."); + + public static final Privilege CREATE_TAGS_PRIVILEGE = Privilege.of( + "CREATE_TAGS", + "Create Tags", + "Create new Tags."); + + public static final Privilege CREATE_DOMAINS_PRIVILEGE = Privilege.of( + "CREATE_DOMAINS", + "Create Domains", + "Create new Domains."); + public static final List PLATFORM_PRIVILEGES = ImmutableList.of( MANAGE_POLICIES_PRIVILEGE, MANAGE_USERS_AND_GROUPS_PRIVILEGE, @@ -85,7 +103,11 @@ public class PoliciesConfig { GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE, MANAGE_ACCESS_TOKENS, MANAGE_TESTS_PRIVILEGE, - MANAGE_GLOSSARIES_PRIVILEGE + MANAGE_GLOSSARIES_PRIVILEGE, + MANAGE_USER_CREDENTIALS_PRIVILEGE, + MANAGE_TAGS_PRIVILEGE, + CREATE_TAGS_PRIVILEGE, + CREATE_DOMAINS_PRIVILEGE ); // Resource Privileges // @@ -150,6 +172,11 @@ public class PoliciesConfig { "Edit All", "The ability to edit any information about an entity. Super user privileges."); + public static final Privilege DELETE_ENTITY_PRIVILEGE = Privilege.of( + "DELETE_ENTITY", + "Delete", + "The ability to delete the delete this entity."); + public static final List COMMON_ENTITY_PRIVILEGES = ImmutableList.of( VIEW_ENTITY_PAGE_PRIVILEGE, EDIT_ENTITY_TAGS_PRIVILEGE, @@ -160,7 +187,8 @@ public class PoliciesConfig { EDIT_ENTITY_STATUS_PRIVILEGE, EDIT_ENTITY_DOMAINS_PRIVILEGE, EDIT_ENTITY_DEPRECATION_PRIVILEGE, - EDIT_ENTITY_PRIVILEGE + EDIT_ENTITY_PRIVILEGE, + DELETE_ENTITY_PRIVILEGE ); // Dataset Privileges @@ -278,7 +306,7 @@ public class PoliciesConfig { "Tags", "Tags indexed by DataHub", ImmutableList.of(VIEW_ENTITY_PAGE_PRIVILEGE, EDIT_ENTITY_OWNERS_PRIVILEGE, EDIT_TAG_COLOR_PRIVILEGE, - EDIT_ENTITY_DOCS_PRIVILEGE, EDIT_ENTITY_PRIVILEGE) + EDIT_ENTITY_DOCS_PRIVILEGE, EDIT_ENTITY_PRIVILEGE, DELETE_ENTITY_PRIVILEGE) ); // Container Privileges @@ -295,7 +323,7 @@ public class PoliciesConfig { "Domains", "Domains created on DataHub", ImmutableList.of(VIEW_ENTITY_PAGE_PRIVILEGE, EDIT_ENTITY_OWNERS_PRIVILEGE, EDIT_ENTITY_DOCS_PRIVILEGE, - EDIT_ENTITY_DOC_LINKS_PRIVILEGE, EDIT_ENTITY_PRIVILEGE) + EDIT_ENTITY_DOC_LINKS_PRIVILEGE, EDIT_ENTITY_PRIVILEGE, DELETE_ENTITY_PRIVILEGE) ); // Glossary Term Privileges diff --git a/settings.gradle b/settings.gradle index ab4eb184517f1d..ff804f610d4b02 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,15 +13,13 @@ include 'metadata-service:restli-client' include 'metadata-service:restli-servlet-impl' include 'metadata-service:graphql-servlet-impl' include 'metadata-service:openapi-servlet' +include 'metadata-service:auth-ranger-impl' include 'metadata-dao-impl:kafka-producer' include 'metadata-events:mxe-avro-1.7' include 'metadata-events:mxe-registration' include 'metadata-events:mxe-schemas' include 'metadata-events:mxe-utils-avro-1.7' include 'metadata-ingestion' -include 'metadata-ingestion-examples:common' -include 'metadata-ingestion-examples:kafka-etl' -include 'metadata-ingestion-examples:mce-cli' include 'metadata-jobs:mae-consumer' include 'metadata-jobs:mce-consumer' include 'metadata-jobs:pe-consumer' @@ -44,6 +42,8 @@ include 'entity-registry:custom-test-model' include 'metadata-integration:java:spark-lineage' include 'metadata-integration:java:datahub-client' include 'metadata-integration:java:datahub-protobuf' -include 'metadata-ingestion-modules:airflow-plugin' include 'ingestion-scheduler' +include 'datahub-ranger-plugin' +include 'metadata-ingestion-modules:airflow-plugin' +include 'smoke-test' include 'ingest-api' diff --git a/smoke-test/.gitignore b/smoke-test/.gitignore index bbad99b9f63857..55142a4a3630f3 100644 --- a/smoke-test/.gitignore +++ b/smoke-test/.gitignore @@ -130,3 +130,4 @@ dmypy.json # Pyre type checker .pyre/ +junit* \ No newline at end of file diff --git a/smoke-test/build.gradle b/smoke-test/build.gradle new file mode 100644 index 00000000000000..ee0ea3c7be384f --- /dev/null +++ b/smoke-test/build.gradle @@ -0,0 +1,40 @@ +apply plugin: 'com.github.node-gradle.node' + +node { + + // If true, it will download node using above parameters. + // If false, it will try to use globally installed node. + if (project.hasProperty('useSystemNode') && project.getProperty('useSystemNode').toBoolean()) { + download = false + } else { + download = true + } + + // Version of node to use. + version = '16.8.0' + + // Version of Yarn to use. + yarnVersion = '1.22.0' + + // Base URL for fetching node distributions (set nodeDistBaseUrl if you have a mirror). + if (project.hasProperty('nodeDistBaseUrl')) { + distBaseUrl = project.getProperty('nodeDistBaseUrl') + } else { + distBaseUrl = 'https://nodejs.org/dist' + } + + // Set the work directory for unpacking node + workDir = file("${project.projectDir}/.gradle/nodejs") + + // Set the work directory for NPM + yarnWorkDir = file("${project.projectDir}/.gradle/yarn") + + // Set the work directory where node_modules should be located + nodeModulesDir = file("${project.projectDir}") + +} + +task yarnInstall(type: YarnTask) { + println "Root directory: ${project.rootDir}"; + args = ['install', '--cwd', "${project.rootDir}/smoke-test/tests/cypress"] +} \ No newline at end of file diff --git a/smoke-test/requirements.txt b/smoke-test/requirements.txt index 111b876e4e92a4..3ff4af3aa14f56 100644 --- a/smoke-test/requirements.txt +++ b/smoke-test/requirements.txt @@ -1,4 +1,7 @@ pytest>=6.2 pytest-dependency>=0.5.1 psutil +tenacity -e ../metadata-ingestion[datahub-rest,datahub-kafka,mysql] +slack-sdk==3.18.1 +aiohttp diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index 379d7465af06b5..141f0c70e17f09 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -22,6 +22,6 @@ pip install -r requirements.txt echo "DATAHUB_VERSION = $DATAHUB_VERSION" DATAHUB_TELEMETRY_ENABLED=false datahub docker quickstart --quickstart-compose-file ../docker/quickstart/docker-compose-without-neo4j.quickstart.yml --dump-logs-on-failure -(cd tests/cypress ; yarn install) +(cd ..; ./gradlew :smoke-test:yarnInstall) -pytest -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml +pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py index 1646f2896b6701..b7d838eeee27e8 100644 --- a/smoke-test/test_e2e.py +++ b/smoke-test/test_e2e.py @@ -1,16 +1,21 @@ import time import urllib +from typing import Any, Optional import pytest import requests - -from datahub.cli.docker import check_local_docker_containers +import tenacity from datahub.ingestion.run.pipeline import Pipeline -from tests.utils import ingest_file_via_rest -GMS_ENDPOINT = "http://localhost:8080" -FRONTEND_ENDPOINT = "http://localhost:9002" -KAFKA_BROKER = "localhost:9092" +from tests.utils import ( + get_frontend_url, + get_gms_url, + get_kafka_broker_url, + get_kafka_schema_registry, + get_sleep_info, + ingest_file_via_rest, + wait_for_healthcheck_util, +) bootstrap_sample_data = "../metadata-ingestion/examples/mce_files/bootstrap_mce.json" usage_sample_data = ( @@ -22,11 +27,12 @@ } kafka_post_ingestion_wait_sec = 60 +sleep_sec, sleep_times = get_sleep_info() + @pytest.fixture(scope="session") def wait_for_healthchecks(): - # Simply assert that everything is healthy, but don't wait. - assert not check_local_docker_containers() + wait_for_healthcheck_util() yield @@ -44,15 +50,58 @@ def frontend_session(wait_for_healthchecks): "Content-Type": "application/json", } data = '{"username":"datahub", "password":"datahub"}' - response = session.post(f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data) + response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data) response.raise_for_status() yield session +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_user_present(urn: str): + response = requests.get( + f"{get_gms_url()}/entities/{urllib.parse.quote(urn)}", + headers={ + **restli_default_headers, + }, + ) + response.raise_for_status() + data = response.json() + + user_key = "com.linkedin.metadata.snapshot.CorpUserSnapshot" + assert data["value"] + assert data["value"][user_key] + assert data["value"][user_key]["urn"] == urn + return data + + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_dataset_present( + urn: str, + aspects: Optional[str] = "datasetProperties", +) -> Any: + response = requests.get( + f"{get_gms_url()}/entitiesV2?ids=List({urllib.parse.quote(urn)})&aspects=List({aspects})", + headers={ + **restli_default_headers, + "X-RestLi-Method": "batch_get", + }, + ) + response.raise_for_status() + res_data = response.json() + assert res_data["results"] + assert res_data["results"][urn] + assert res_data["results"][urn]["aspects"]["datasetProperties"] + return res_data + + @pytest.mark.dependency(depends=["test_healthchecks"]) def test_ingestion_via_rest(wait_for_healthchecks): ingest_file_via_rest(bootstrap_sample_data) + _ensure_user_present(urn="urn:li:corpuser:datahub") @pytest.mark.dependency(depends=["test_healthchecks"]) @@ -72,7 +121,8 @@ def test_ingestion_via_kafka(wait_for_healthchecks): "type": "datahub-kafka", "config": { "connection": { - "bootstrap": KAFKA_BROKER, + "bootstrap": get_kafka_broker_url(), + "schema_registry_url": get_kafka_schema_registry(), } }, }, @@ -80,6 +130,9 @@ def test_ingestion_via_kafka(wait_for_healthchecks): ) pipeline.run() pipeline.raise_from_status() + _ensure_dataset_present( + "urn:li:dataset:(urn:li:dataPlatform:bigquery,bigquery-public-data.covid19_geotab_mobility_impact.us_border_wait_times,PROD)" + ) # Since Kafka emission is asynchronous, we must wait a little bit so that # the changes are actually processed. @@ -102,20 +155,7 @@ def test_run_ingestion(wait_for_healthchecks): def test_gms_get_user(): username = "jdoe" urn = f"urn:li:corpuser:{username}" - response = requests.get( - f"{GMS_ENDPOINT}/entities/{urllib.parse.quote(urn)}", - headers={ - **restli_default_headers, - }, - ) - response.raise_for_status() - data = response.json() - - assert data["value"] - assert data["value"]["com.linkedin.metadata.snapshot.CorpUserSnapshot"] - assert ( - data["value"]["com.linkedin.metadata.snapshot.CorpUserSnapshot"]["urn"] == urn - ) + _ensure_user_present(urn=urn) @pytest.mark.parametrize( @@ -145,7 +185,7 @@ def test_gms_get_dataset(platform, dataset_name, env): urn = f"urn:li:dataset:({platform},{dataset_name},{env})" response = requests.get( - f"{GMS_ENDPOINT}/entities/{urllib.parse.quote(urn)}", + f"{get_gms_url()}/entities/{urllib.parse.quote(urn)}", headers={ **restli_default_headers, "X-RestLi-Method": "get", @@ -171,25 +211,12 @@ def test_gms_batch_get_v2(): urn1 = f"urn:li:dataset:({platform},{name_1},{env})" urn2 = f"urn:li:dataset:({platform},{name_2},{env})" - response = requests.get( - f"{GMS_ENDPOINT}/entitiesV2?ids=List({urllib.parse.quote(urn1)},{urllib.parse.quote(urn2)})&aspects=List(datasetProperties,ownership)", - headers={ - **restli_default_headers, - "X-RestLi-Method": "batch_get", - }, - ) - response.raise_for_status() - res_data = response.json() + resp1 = _ensure_dataset_present(urn1, aspects="datasetProperties,ownership") + assert resp1["results"][urn1]["aspects"]["ownership"] - # Verify both urns exist and have correct aspects - assert res_data["results"] - assert res_data["results"][urn1] - assert res_data["results"][urn1]["aspects"]["datasetProperties"] - assert res_data["results"][urn1]["aspects"]["ownership"] - assert res_data["results"][urn2] - assert res_data["results"][urn2]["aspects"]["datasetProperties"] + resp2 = _ensure_dataset_present(urn2, aspects="datasetProperties,ownership") assert ( - "ownership" not in res_data["results"][urn2]["aspects"] + "ownership" not in resp2["results"][urn2]["aspects"] ) # Aspect does not exist. @@ -206,7 +233,7 @@ def test_gms_search_dataset(query, min_expected_results): json = {"input": f"{query}", "entity": "dataset", "start": 0, "count": 10} print(json) response = requests.post( - f"{GMS_ENDPOINT}/entities?action=search", + f"{get_gms_url()}/entities?action=search", headers=restli_default_headers, json=json, ) @@ -231,7 +258,7 @@ def test_gms_search_across_entities(query, min_expected_results): json = {"input": f"{query}", "entities": [], "start": 0, "count": 10} print(json) response = requests.post( - f"{GMS_ENDPOINT}/entities?action=searchAcrossEntities", + f"{get_gms_url()}/entities?action=searchAcrossEntities", headers=restli_default_headers, json=json, ) @@ -246,7 +273,7 @@ def test_gms_search_across_entities(query, min_expected_results): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_usage_fetch(): response = requests.post( - f"{GMS_ENDPOINT}/usageStats?action=queryRange", + f"{get_gms_url()}/usageStats?action=queryRange", headers=restli_default_headers, json={ "resource": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_schema.excess_deaths_derived,PROD)", @@ -304,7 +331,7 @@ def test_frontend_browse_datasets(frontend_session): "variables": {"input": {"type": "DATASET", "path": ["prod"]}}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -347,7 +374,7 @@ def test_frontend_search_datasets(frontend_session, query, min_expected_results) }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -390,7 +417,7 @@ def test_frontend_search_across_entities(frontend_session, query, min_expected_r }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -426,7 +453,7 @@ def test_frontend_user_info(frontend_session): }""", "variables": {"urn": urn}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -475,7 +502,7 @@ def test_frontend_datasets(frontend_session, platform, dataset_name, env): "variables": {"urn": urn}, } # Basic dataset info. - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -490,7 +517,7 @@ def test_frontend_datasets(frontend_session, platform, dataset_name, env): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_ingest_with_system_metadata(): response = requests.post( - f"{GMS_ENDPOINT}/entities?action=ingest", + f"{get_gms_url()}/entities?action=ingest", headers=restli_default_headers, json={ "entity": { @@ -523,7 +550,7 @@ def test_ingest_with_system_metadata(): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_ingest_with_blank_system_metadata(): response = requests.post( - f"{GMS_ENDPOINT}/entities?action=ingest", + f"{get_gms_url()}/entities?action=ingest", headers=restli_default_headers, json={ "entity": { @@ -553,7 +580,7 @@ def test_ingest_with_blank_system_metadata(): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_ingest_without_system_metadata(): response = requests.post( - f"{GMS_ENDPOINT}/entities?action=ingest", + f"{get_gms_url()}/entities?action=ingest", headers=restli_default_headers, json={ "entity": { @@ -579,227 +606,6 @@ def test_ingest_without_system_metadata(): response.raise_for_status() -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_frontend_list_policies(frontend_session): - - json = { - "query": """query listPolicies($input: ListPoliciesInput!) {\n - listPolicies(input: $input) {\n - start\n - count\n - total\n - policies {\n - urn\n - type\n - name\n - description\n - state\n - resources {\n - type\n - allResources\n - resources\n - }\n - privileges\n - actors {\n - users\n - groups\n - allUsers\n - allGroups\n - resourceOwners\n - }\n - editable\n - }\n - }\n - }""", - "variables": { - "input": { - "start": "0", - "count": "20", - } - }, - } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["listPolicies"] - assert res_data["data"]["listPolicies"]["start"] == 0 - assert res_data["data"]["listPolicies"]["count"] > 0 - assert len(res_data["data"]["listPolicies"]["policies"]) > 0 - - -@pytest.mark.dependency( - depends=["test_healthchecks", "test_run_ingestion", "test_frontend_list_policies"] -) -def test_frontend_update_policy(frontend_session): - - json = { - "query": """mutation updatePolicy($urn: String!, $input: PolicyUpdateInput!) {\n - updatePolicy(urn: $urn, input: $input) }""", - "variables": { - "urn": "urn:li:dataHubPolicy:7", - "input": { - "type": "PLATFORM", - "state": "INACTIVE", - "name": "Updated Platform Policy", - "description": "My Metadaata Policy", - "privileges": ["MANAGE_POLICIES"], - "actors": { - "users": ["urn:li:corpuser:datahub"], - "resourceOwners": False, - "allUsers": False, - "allGroups": False, - }, - }, - }, - } - - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["updatePolicy"] - assert res_data["data"]["updatePolicy"] == "urn:li:dataHubPolicy:7" - - -@pytest.mark.dependency( - depends=[ - "test_healthchecks", - "test_run_ingestion", - "test_frontend_list_policies", - "test_frontend_update_policy", - ] -) -def test_frontend_delete_policy(frontend_session): - - json = { - "query": """mutation deletePolicy($urn: String!) {\n - deletePolicy(urn: $urn) }""", - "variables": {"urn": "urn:li:dataHubPolicy:7"}, - } - - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) - response.raise_for_status() - res_data = response.json() - - # Now verify the policy has been removed. - json = { - "query": """query listPolicies($input: ListPoliciesInput!) {\n - listPolicies(input: $input) {\n - start\n - count\n - total\n - policies {\n - urn\n - }\n - }\n - }""", - "variables": { - "input": { - "start": "0", - "count": "20", - } - }, - } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["listPolicies"] - - # Verify that the URN is no longer in the list - result = filter( - lambda x: x["urn"] == "urn:li:dataHubPolicy:7", - res_data["data"]["listPolicies"]["policies"], - ) - assert len(list(result)) == 0 - - -@pytest.mark.dependency( - depends=[ - "test_healthchecks", - "test_run_ingestion", - "test_frontend_list_policies", - "test_frontend_delete_policy", - ] -) -def test_frontend_create_policy(frontend_session): - - # Policy tests are not idempotent. If you rerun this test it will be wrong. - json = { - "query": """mutation createPolicy($input: PolicyUpdateInput!) {\n - createPolicy(input: $input) }""", - "variables": { - "input": { - "type": "METADATA", - "name": "Test Metadata Policy", - "description": "My Metadaata Policy", - "state": "ACTIVE", - "resources": {"type": "dataset", "allResources": True}, - "privileges": ["EDIT_ENTITY_TAGS"], - "actors": { - "users": ["urn:li:corpuser:datahub"], - "resourceOwners": False, - "allUsers": False, - "allGroups": False, - }, - } - }, - } - - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["createPolicy"] - - new_urn = res_data["data"]["createPolicy"] - - # Sleep for eventual consistency - time.sleep(3) - - # Now verify the policy has been added. - json = { - "query": """query listPolicies($input: ListPoliciesInput!) {\n - listPolicies(input: $input) {\n - start\n - count\n - total\n - policies {\n - urn\n - }\n - }\n - }""", - "variables": { - "input": { - "start": "0", - "count": "20", - } - }, - } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["listPolicies"] - - # Verify that the URN appears in the list - result = filter( - lambda x: x["urn"] == new_urn, res_data["data"]["listPolicies"]["policies"] - ) - assert len(list(result)) == 1 - - @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_frontend_app_config(frontend_session): @@ -831,7 +637,7 @@ def test_frontend_app_config(frontend_session): }""" } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -865,13 +671,14 @@ def test_frontend_me_query(frontend_session): viewAnalytics managePolicies manageIdentities - generatePersonalAccessTokens + manageUserCredentials + generatePersonalAccessTokens }\n }\n }""" } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -880,6 +687,7 @@ def test_frontend_me_query(frontend_session): assert res_data["data"]["me"]["corpUser"]["urn"] == "urn:li:corpuser:datahub" assert res_data["data"]["me"]["platformPrivileges"]["viewAnalytics"] is True assert res_data["data"]["me"]["platformPrivileges"]["managePolicies"] is True + assert res_data["data"]["me"]["platformPrivileges"]["manageUserCredentials"] is True assert res_data["data"]["me"]["platformPrivileges"]["manageIdentities"] is True assert ( res_data["data"]["me"]["platformPrivileges"]["generatePersonalAccessTokens"] @@ -913,7 +721,7 @@ def test_list_users(frontend_session): } }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -953,7 +761,7 @@ def test_list_groups(frontend_session): } }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -977,14 +785,14 @@ def test_add_remove_members_from_group(frontend_session): "query": """query corpUser($urn: String!) {\n corpUser(urn: $urn) {\n urn\n - relationships(input: { types: ["IsMemberOfGroup"], direction: OUTGOING, start: 0, count: 1 }) {\n + relationships(input: { types: ["IsMemberOfNativeGroup"], direction: OUTGOING, start: 0, count: 1 }) {\n total\n }\n }\n }""", "variables": {"urn": "urn:li:corpuser:jdoe"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -1005,7 +813,7 @@ def test_add_remove_members_from_group(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() # Sleep for edge store to be updated. Not ideal! @@ -1016,14 +824,14 @@ def test_add_remove_members_from_group(frontend_session): "query": """query corpUser($urn: String!) {\n corpUser(urn: $urn) {\n urn\n - relationships(input: { types: ["IsMemberOfGroup"], direction: OUTGOING, start: 0, count: 1 }) {\n + relationships(input: { types: ["IsMemberOfNativeGroup"], direction: OUTGOING, start: 0, count: 1 }) {\n total\n }\n }\n }""", "variables": {"urn": "urn:li:corpuser:jdoe"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -1045,7 +853,7 @@ def test_add_remove_members_from_group(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() # Sleep for edge store to be updated. Not ideal! @@ -1056,14 +864,14 @@ def test_add_remove_members_from_group(frontend_session): "query": """query corpUser($urn: String!) {\n corpUser(urn: $urn) {\n urn\n - relationships(input: { types: ["IsMemberOfGroup"], direction: OUTGOING, start: 0, count: 1 }) {\n + relationships(input: { types: ["IsMemberOfNativeGroup"], direction: OUTGOING, start: 0, count: 1 }) {\n total\n }\n }\n }""", "variables": {"urn": "urn:li:corpuser:jdoe"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -1073,9 +881,7 @@ def test_add_remove_members_from_group(frontend_session): assert res_data["data"]["corpUser"]["relationships"]["total"] == 0 -@pytest.mark.dependency( - depends=["test_healthchecks", "test_run_ingestion"] -) +@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_update_corp_group_properties(frontend_session): group_urn = "urn:li:corpGroup:bfoo" @@ -1085,20 +891,20 @@ def test_update_corp_group_properties(frontend_session): "query": """mutation updateCorpGroupProperties($urn: String!, $input: CorpGroupUpdateInput!) {\n updateCorpGroupProperties(urn: $urn, input: $input) { urn } }""", "variables": { - "urn": group_urn, - "input": { - "description": "My test description", - "slack": "test_group_slack", - "email": "test_group_email@email.com", - }, + "urn": group_urn, + "input": { + "description": "My test description", + "slack": "test_group_slack", + "email": "test_group_email@email.com", + }, }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() print(res_data) - assert "error" not in res_data + assert "errors" not in res_data assert res_data["data"]["updateCorpGroupProperties"] is not None # Verify the description has been updated @@ -1115,40 +921,41 @@ def test_update_corp_group_properties(frontend_session): }""", "variables": {"urn": group_urn}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() assert res_data - assert "error" not in res_data + assert "errors" not in res_data assert res_data["data"] assert res_data["data"]["corpGroup"] assert res_data["data"]["corpGroup"]["editableProperties"] assert res_data["data"]["corpGroup"]["editableProperties"] == { - "description": "My test description", - "slack": "test_group_slack", - "email": "test_group_email@email.com" + "description": "My test description", + "slack": "test_group_slack", + "email": "test_group_email@email.com", } # Reset the editable properties json = { - "query": """mutation updateCorpGroupProperties($urn: String!, $input: UpdateCorpGroupPropertiesInput!) {\n - updateCorpGroupProperties(urn: $urn, input: $input) }""", + "query": """mutation updateCorpGroupProperties($urn: String!, $input: CorpGroupUpdateInput!) {\n + updateCorpGroupProperties(urn: $urn, input: $input) { urn } }""", "variables": { - "urn": group_urn, - "input": { - "description": "", - "slack": "", - "email": "" - }, + "urn": group_urn, + "input": {"description": "", "slack": "", "email": ""}, }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() + @pytest.mark.dependency( - depends=["test_healthchecks", "test_run_ingestion", "test_update_corp_group_properties"] + depends=[ + "test_healthchecks", + "test_run_ingestion", + "test_update_corp_group_properties", + ] ) def test_update_corp_group_description(frontend_session): @@ -1159,18 +966,15 @@ def test_update_corp_group_description(frontend_session): "query": """mutation updateDescription($input: DescriptionUpdateInput!) {\n updateDescription(input: $input) }""", "variables": { - "input": { - "description": "My test description", - "resourceUrn": group_urn - }, + "input": {"description": "My test description", "resourceUrn": group_urn}, }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() print(res_data) - assert "error" not in res_data + assert "errors" not in res_data assert res_data["data"]["updateDescription"] is True # Verify the description has been updated @@ -1185,32 +989,33 @@ def test_update_corp_group_description(frontend_session): }""", "variables": {"urn": group_urn}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() assert res_data - assert "error" not in res_data + assert "errors" not in res_data assert res_data["data"] assert res_data["data"]["corpGroup"] assert res_data["data"]["corpGroup"]["editableProperties"] - assert res_data["data"]["corpGroup"]["editableProperties"]["description"] == "My test description" + assert ( + res_data["data"]["corpGroup"]["editableProperties"]["description"] + == "My test description" + ) # Reset Corp Group Description json = { "query": """mutation updateDescription($input: DescriptionUpdateInput!) {\n updateDescription(input: $input) }""", "variables": { - "input": { - "description": "", - "resourceUrn": group_urn - }, + "input": {"description": "", "resourceUrn": group_urn}, }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() + @pytest.mark.dependency( depends=[ "test_healthchecks", @@ -1227,7 +1032,7 @@ def test_remove_user(frontend_session): "variables": {"urn": "urn:li:corpuser:jdoe"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() json = { @@ -1241,12 +1046,12 @@ def test_remove_user(frontend_session): }""", "variables": {"urn": "urn:li:corpuser:jdoe"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() assert res_data - assert "error" not in res_data + assert "errors" not in res_data assert res_data["data"] assert res_data["data"]["corpUser"] assert res_data["data"]["corpUser"]["properties"] is None @@ -1268,7 +1073,7 @@ def test_remove_group(frontend_session): "variables": {"urn": "urn:li:corpGroup:bfoo"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() json = { @@ -1282,7 +1087,7 @@ def test_remove_group(frontend_session): }""", "variables": {"urn": "urn:li:corpGroup:bfoo"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -1314,7 +1119,7 @@ def test_create_group(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() json = { @@ -1328,7 +1133,7 @@ def test_create_group(frontend_session): }""", "variables": {"urn": "urn:li:corpGroup:test-id"}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -1355,7 +1160,7 @@ def test_home_page_recommendations(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() print(res_data) @@ -1363,7 +1168,7 @@ def test_home_page_recommendations(frontend_session): assert res_data assert res_data["data"] assert res_data["data"]["listRecommendations"] - assert "error" not in res_data + assert "errors" not in res_data assert ( len(res_data["data"]["listRecommendations"]["modules"]) > min_expected_recommendation_modules @@ -1376,7 +1181,7 @@ def test_search_results_recommendations(frontend_session): # This test simply ensures that the recommendations endpoint does not return an error. json = { "query": """query listRecommendations($input: ListRecommendationsInput!) {\n - listRecommendations(input: $input) { modules { title } }""", + listRecommendations(input: $input) { modules { title } } }""", "variables": { "input": { "userUrn": "urn:li:corpuser:datahub", @@ -1389,12 +1194,12 @@ def test_search_results_recommendations(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() assert res_data - assert "error" not in res_data + assert "errors" not in res_data @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @@ -1416,19 +1221,21 @@ def test_generate_personal_access_token(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] assert res_data["data"]["getAccessToken"]["accessToken"] is not None - assert "error" not in res_data + assert "errors" not in res_data # Test unauthenticated case json = { "query": """query getAccessToken($input: GetAccessTokenInput!) {\n - accessToken\n + getAccessToken(input: $input) {\n + accessToken\n + }\n }""", "variables": { "input": { @@ -1439,9 +1246,211 @@ def test_generate_personal_access_token(frontend_session): }, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() assert res_data assert "errors" in res_data # Assert the request fails + + +@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) +def test_native_user_endpoints(frontend_session): + # Sign up tests + + # Test getting the invite token + get_invite_token_json = { + "query": """query getNativeUserInviteToken {\n + getNativeUserInviteToken{\n + inviteToken\n + }\n + }""" + } + + get_invite_token_response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=get_invite_token_json + ) + get_invite_token_response.raise_for_status() + get_invite_token_res_data = get_invite_token_response.json() + + assert get_invite_token_res_data + assert get_invite_token_res_data["data"] + invite_token = get_invite_token_res_data["data"]["getNativeUserInviteToken"][ + "inviteToken" + ] + assert invite_token is not None + assert "errors" not in get_invite_token_res_data + + # Pass the invite token when creating the user + sign_up_json = { + "fullName": "Test User", + "email": "test@email.com", + "password": "password", + "title": "Date Engineer", + "inviteToken": invite_token, + } + + sign_up_response = frontend_session.post( + f"{get_frontend_url()}/signUp", json=sign_up_json + ) + assert sign_up_response + assert "errors" not in sign_up_response + + # Creating the same user again fails + same_user_sign_up_response = frontend_session.post( + f"{get_frontend_url()}/signUp", json=sign_up_json + ) + assert not same_user_sign_up_response + + # Test that a bad invite token leads to failed sign up + bad_sign_up_json = { + "fullName": "Test2 User", + "email": "test2@email.com", + "password": "password", + "title": "Date Engineer", + "inviteToken": "invite_token", + } + bad_sign_up_response = frontend_session.post( + f"{get_frontend_url()}/signUp", json=bad_sign_up_json + ) + assert not bad_sign_up_response + + frontend_session.cookies.clear() + + # Reset credentials tests + + # Log in as root again + headers = { + "Content-Type": "application/json", + } + root_login_data = '{"username":"datahub", "password":"datahub"}' + frontend_session.post( + f"{get_frontend_url()}/logIn", headers=headers, data=root_login_data + ) + + # Test creating the password reset token + create_reset_token_json = { + "query": """mutation createNativeUserResetToken($input: CreateNativeUserResetTokenInput!) {\n + createNativeUserResetToken(input: $input) {\n + resetToken\n + }\n + }""", + "variables": {"input": {"userUrn": "urn:li:corpuser:test@email.com"}}, + } + + create_reset_token_response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=create_reset_token_json + ) + create_reset_token_response.raise_for_status() + create_reset_token_res_data = create_reset_token_response.json() + + assert create_reset_token_res_data + assert create_reset_token_res_data["data"] + reset_token = create_reset_token_res_data["data"]["createNativeUserResetToken"][ + "resetToken" + ] + assert reset_token is not None + assert "errors" not in create_reset_token_res_data + + # Pass the reset token when resetting credentials + reset_credentials_json = { + "email": "test@email.com", + "password": "password", + "resetToken": reset_token, + } + + reset_credentials_response = frontend_session.post( + f"{get_frontend_url()}/resetNativeUserCredentials", json=reset_credentials_json + ) + assert reset_credentials_response + assert "errors" not in reset_credentials_response + + # Test that a bad reset token leads to failed response + bad_user_reset_credentials_json = { + "email": "test@email.com", + "password": "password", + "resetToken": "reset_token", + } + bad_reset_credentials_response = frontend_session.post( + f"{get_frontend_url()}/resetNativeUserCredentials", + json=bad_user_reset_credentials_json, + ) + assert not bad_reset_credentials_response + + # Test that only a native user can reset their password + jaas_user_reset_credentials_json = { + "email": "datahub", + "password": "password", + "resetToken": reset_token, + } + jaas_user_reset_credentials_response = frontend_session.post( + f"{get_frontend_url()}/resetNativeUserCredentials", + json=jaas_user_reset_credentials_json, + ) + assert not jaas_user_reset_credentials_response + + # Tests that unauthenticated users can't invite users or send reset password links + + native_user_frontend_session = requests.Session() + + native_user_login_data = '{"username":"test@email.com", "password":"password"}' + native_user_frontend_session.post( + f"{get_frontend_url()}/logIn", headers=headers, data=native_user_login_data + ) + + unauthenticated_get_invite_token_response = native_user_frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=get_invite_token_json + ) + unauthenticated_get_invite_token_response.raise_for_status() + unauthenticated_get_invite_token_res_data = ( + unauthenticated_get_invite_token_response.json() + ) + + assert unauthenticated_get_invite_token_res_data + assert "errors" in unauthenticated_get_invite_token_res_data + assert unauthenticated_get_invite_token_res_data["data"] + assert ( + unauthenticated_get_invite_token_res_data["data"]["getNativeUserInviteToken"] + is None + ) + + unauthenticated_create_reset_token_json = { + "query": """mutation createNativeUserResetToken($input: CreateNativeUserResetTokenInput!) {\n + createNativeUserResetToken(input: $input) {\n + resetToken\n + }\n + }""", + "variables": {"input": {"userUrn": "urn:li:corpuser:test@email.com"}}, + } + + unauthenticated_create_reset_token_response = native_user_frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", + json=unauthenticated_create_reset_token_json, + ) + unauthenticated_create_reset_token_response.raise_for_status() + unauthenticated_create_reset_token_res_data = ( + unauthenticated_create_reset_token_response.json() + ) + + assert unauthenticated_create_reset_token_res_data + assert "errors" in unauthenticated_create_reset_token_res_data + assert unauthenticated_create_reset_token_res_data["data"] + assert ( + unauthenticated_create_reset_token_res_data["data"][ + "createNativeUserResetToken" + ] + is None + ) + + # cleanup steps + json = { + "query": """mutation removeUser($urn: String!) {\n + removeUser(urn: $urn) }""", + "variables": {"urn": "urn:li:corpuser:test@email.com"}, + } + + remove_user_response = native_user_frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=json + ) + remove_user_response.raise_for_status() + assert "errors" not in remove_user_response diff --git a/smoke-test/test_rapid.py b/smoke-test/test_rapid.py index 0219154bdb51d3..e6cd421e5cc915 100644 --- a/smoke-test/test_rapid.py +++ b/smoke-test/test_rapid.py @@ -1,25 +1,20 @@ import time -import urllib -from typing import Any, Dict, Optional, cast import pytest import requests -from datahub.cli.docker import check_local_docker_containers -from datahub.ingestion.run.pipeline import Pipeline -from datahub.ingestion.source.state.checkpoint import Checkpoint -from tests.utils import ingest_file_via_rest +from tests.utils import get_frontend_url, ingest_file_via_rest, wait_for_healthcheck_util bootstrap_small = "test_resources/bootstrap_single.json" bootstrap_small_2 = "test_resources/bootstrap_single2.json" -FRONTEND_ENDPOINT = "http://localhost:9002" + @pytest.fixture(scope="session") def wait_for_healthchecks(): - # Simply assert that everything is healthy, but don't wait. - assert not check_local_docker_containers() + wait_for_healthcheck_util() yield + @pytest.fixture(scope="session") def frontend_session(wait_for_healthchecks): session = requests.Session() @@ -28,19 +23,18 @@ def frontend_session(wait_for_healthchecks): "Content-Type": "application/json", } data = '{"username":"datahub", "password":"datahub"}' - response = session.post( - f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data - ) + response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data) response.raise_for_status() yield session + def test_ingestion_via_rest_rapid(frontend_session, wait_for_healthchecks): ingest_file_via_rest(bootstrap_small) ingest_file_via_rest(bootstrap_small_2) - urn = f"urn:li:dataset:(urn:li:dataPlatform:testPlatform,testDataset,PROD)" + urn = "urn:li:dataset:(urn:li:dataPlatform:testPlatform,testDataset,PROD)" json = { - "query": """query getDataset($urn: String!) {\n + "query": """query getDataset($urn: String!) {\n dataset(urn: $urn) {\n urn\n name\n @@ -70,15 +64,11 @@ def test_ingestion_via_rest_rapid(frontend_session, wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": urn - } - } + "variables": {"urn": urn}, + } # time.sleep(2) - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() diff --git a/smoke-test/tests/__init__.py b/smoke-test/tests/__init__.py index 456b9b5d44f052..5ac137668288a9 100644 --- a/smoke-test/tests/__init__.py +++ b/smoke-test/tests/__init__.py @@ -1,4 +1,4 @@ import os # Disable telemetry -os.putenv("DATAHUB_TELEMETRY_ENABLED", "false") +os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false" diff --git a/smoke-test/tests/assertions/assertions_test.py b/smoke-test/tests/assertions/assertions_test.py index 99ae7110de0bd7..5e749d2214fd7a 100644 --- a/smoke-test/tests/assertions/assertions_test.py +++ b/smoke-test/tests/assertions/assertions_test.py @@ -4,8 +4,6 @@ import pytest import requests - -from datahub.cli.docker import check_local_docker_containers from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext, RecordEnvelope @@ -25,11 +23,7 @@ PartitionSpecClass, PartitionTypeClass, ) -from tests.utils import ingest_file_via_rest -from tests.utils import delete_urns_from_file - - -GMS_ENDPOINT = "http://localhost:8080" +from tests.utils import delete_urns_from_file, get_gms_url, ingest_file_via_rest, wait_for_healthcheck_util restli_default_headers = { "X-RestLi-Protocol-Version": "2.0.0", @@ -68,7 +62,6 @@ def create_test_data(test_file): 1643880726874, 1643880726875, ] - msg_ids = [] # The assertion run event attached to the dataset mcp2 = MetadataChangeProposalWrapper( entityType="assertion", @@ -238,8 +231,7 @@ def generate_test_data(tmp_path_factory): @pytest.fixture(scope="session") def wait_for_healthchecks(generate_test_data): - # Simply assert that everything is healthy, but don't wait. - assert not check_local_docker_containers() + wait_for_healthcheck_util() yield @@ -253,6 +245,7 @@ def test_healthchecks(wait_for_healthchecks): def test_run_ingestion(generate_test_data): ingest_file_via_rest(generate_test_data) + @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_get_latest_assertions_results_by_partition(): urn = make_dataset_urn("postgres", "foo") @@ -294,7 +287,7 @@ def test_gms_get_latest_assertions_results_by_partition(): } ) response = requests.post( - f"{GMS_ENDPOINT}/analytics?action=getTimeseriesStats", + f"{get_gms_url()}/analytics?action=getTimeseriesStats", data=query, headers=restli_default_headers, ) @@ -325,7 +318,7 @@ def test_gms_get_assertions_on_dataset(): """lists all assertion urns including those which may not have executed""" urn = make_dataset_urn("postgres", "foo") response = requests.get( - f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(urn)}&types=Asserts" + f"{get_gms_url()}/relationships?direction=INCOMING&urn={urllib.parse.quote(urn)}&types=Asserts" ) response.raise_for_status() @@ -339,7 +332,7 @@ def test_gms_get_assertions_on_dataset_field(): dataset_urn = make_dataset_urn("postgres", "foo") field_urn = make_schema_field_urn(dataset_urn, "col1") response = requests.get( - f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(field_urn)}&types=Asserts" + f"{get_gms_url()}/relationships?direction=INCOMING&urn={urllib.parse.quote(field_urn)}&types=Asserts" ) response.raise_for_status() @@ -351,7 +344,7 @@ def test_gms_get_assertions_on_dataset_field(): def test_gms_get_assertion_info(): assertion_urn = "urn:li:assertion:2d3b06a6e77e1f24adc9860a05ea089b" response = requests.get( - f"{GMS_ENDPOINT}/aspects/{urllib.parse.quote(assertion_urn)}\ + f"{get_gms_url()}/aspects/{urllib.parse.quote(assertion_urn)}\ ?aspect=assertionInfo&version=0", headers=restli_default_headers, ) @@ -364,4 +357,4 @@ def test_gms_get_assertion_info(): assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["type"] == "DATASET" assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"][ "scope" - ] \ No newline at end of file + ] diff --git a/smoke-test/tests/cli/datahub_graph_test.py b/smoke-test/tests/cli/datahub_graph_test.py index a4b01e452217f7..371edd66563b4a 100644 --- a/smoke-test/tests/cli/datahub_graph_test.py +++ b/smoke-test/tests/cli/datahub_graph_test.py @@ -1,14 +1,8 @@ -import time - import pytest from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph -from datahub.metadata.schema_classes import SchemaMetadataClass, KafkaSchemaClass -from tests.utils import ( - FRONTEND_ENDPOINT, - GMS_ENDPOINT, - delete_urns_from_file, - ingest_file_via_rest, -) +from datahub.metadata.schema_classes import KafkaSchemaClass, SchemaMetadataClass +from tests.utils import delete_urns_from_file, ingest_file_via_rest, get_gms_url + @pytest.fixture(scope="module", autouse=False) @@ -28,7 +22,7 @@ def test_healthchecks(wait_for_healthchecks): @pytest.mark.dependency(depends=["test_healthchecks"]) def test_get_aspect_v2(frontend_session, ingest_cleanup_data): - graph: DataHubGraph = DataHubGraph(DatahubClientConfig()) + graph: DataHubGraph = DataHubGraph(DatahubClientConfig(server=get_gms_url())) urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-rollback,PROD)" schema_metadata: SchemaMetadataClass = graph.get_aspect_v2( urn, aspect="schemaMetadata", aspect_type=SchemaMetadataClass @@ -38,5 +32,7 @@ def test_get_aspect_v2(frontend_session, ingest_cleanup_data): assert schema_metadata.platform == "urn:li:dataPlatform:kafka" assert isinstance(schema_metadata.platformSchema, KafkaSchemaClass) k_schema: KafkaSchemaClass = schema_metadata.platformSchema - assert k_schema.documentSchema == "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" - + assert ( + k_schema.documentSchema + == '{"type":"record","name":"SampleKafkaSchema","namespace":"com.linkedin.dataset","doc":"Sample Kafka dataset","fields":[{"name":"field_foo","type":["string"]},{"name":"field_bar","type":["boolean"]}]}' + ) diff --git a/smoke-test/tests/conftest.py b/smoke-test/tests/conftest.py index 7d0f3c159864b1..b6561f318be42e 100644 --- a/smoke-test/tests/conftest.py +++ b/smoke-test/tests/conftest.py @@ -1,22 +1,21 @@ import os -import time import pytest import requests -import urllib -from datahub.cli.docker import check_local_docker_containers -from datahub.ingestion.run.pipeline import Pipeline -from tests.utils import FRONTEND_ENDPOINT + +from tests.utils import get_frontend_url, wait_for_healthcheck_util +from tests.test_result_msg import send_message # Disable telemetry -os.putenv("DATAHUB_TELEMETRY_ENABLED", "false") +os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false" + @pytest.fixture(scope="session") def wait_for_healthchecks(): - # Simply assert that everything is healthy, but don't wait. - assert not check_local_docker_containers() + wait_for_healthcheck_util() yield + @pytest.fixture(scope="session") def frontend_session(wait_for_healthchecks): session = requests.Session() @@ -25,15 +24,19 @@ def frontend_session(wait_for_healthchecks): "Content-Type": "application/json", } data = '{"username":"datahub", "password":"datahub"}' - response = session.post( - f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data - ) + response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data) response.raise_for_status() yield session + # TODO: Determine whether we need this or not. @pytest.mark.dependency() def test_healthchecks(wait_for_healthchecks): # Call to wait_for_healthchecks fixture will do the actual functionality. - pass \ No newline at end of file + pass + + +def pytest_sessionfinish(session, exitstatus): + """ whole test run finishes. """ + send_message(exitstatus) diff --git a/smoke-test/tests/containers/containers_test.py b/smoke-test/tests/containers/containers_test.py index daa6b2bbc7bd85..575e3def6cf232 100644 --- a/smoke-test/tests/containers/containers_test.py +++ b/smoke-test/tests/containers/containers_test.py @@ -1,9 +1,6 @@ import pytest -import time -from tests.utils import FRONTEND_ENDPOINT -from tests.utils import GMS_ENDPOINT -from tests.utils import ingest_file_via_rest -from tests.utils import delete_urns_from_file +from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest + @pytest.fixture(scope="module", autouse=False) def ingest_cleanup_data(request): @@ -13,11 +10,13 @@ def ingest_cleanup_data(request): print("removing containers test data") delete_urns_from_file("tests/containers/data.json") + @pytest.mark.dependency() def test_healthchecks(wait_for_healthchecks): # Call to wait_for_healthchecks fixture will do the actual functionality. pass + @pytest.mark.dependency(depends=["test_healthchecks"]) def test_get_full_container(frontend_session, ingest_cleanup_data): @@ -96,13 +95,11 @@ def test_get_full_container(frontend_session, ingest_cleanup_data): }\n }\n }""", - "variables": { - "urn": container_urn - } + "variables": {"urn": container_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_container_json + f"{get_frontend_url()}/api/v2/graphql", json=get_container_json ) response.raise_for_status() res_data = response.json() @@ -119,12 +116,15 @@ def test_get_full_container(frontend_session, ingest_cleanup_data): assert container["properties"]["name"] == container_name assert container["properties"]["description"] == container_description assert container["subTypes"]["typeNames"][0] == "Schema" - assert container["editableProperties"]["description"] == editable_container_description + assert ( + container["editableProperties"]["description"] == editable_container_description + ) assert container["ownership"] is None assert container["institutionalMemory"] is None assert container["tags"] is None assert container["glossaryTerms"] is None + @pytest.mark.dependency(depends=["test_healthchecks", "test_get_full_container"]) def test_get_parent_container(frontend_session, ingest_cleanup_data): @@ -143,13 +143,11 @@ def test_get_parent_container(frontend_session, ingest_cleanup_data): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=get_dataset_json ) response.raise_for_status() res_data = response.json() @@ -162,6 +160,7 @@ def test_get_parent_container(frontend_session, ingest_cleanup_data): dataset = res_data["data"]["dataset"] assert dataset["container"]["properties"]["name"] == "datahub_schema" + @pytest.mark.dependency(depends=["test_healthchecks", "test_get_full_container"]) def test_update_container(frontend_session, ingest_cleanup_data): @@ -175,14 +174,14 @@ def test_update_container(frontend_session, ingest_cleanup_data): }""", "variables": { "input": { - "tagUrn": new_tag, - "resourceUrn": container_urn, + "tagUrn": new_tag, + "resourceUrn": container_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_tag_json + f"{get_frontend_url()}/api/v2/graphql", json=add_tag_json ) response.raise_for_status() res_data = response.json() @@ -199,14 +198,14 @@ def test_update_container(frontend_session, ingest_cleanup_data): }""", "variables": { "input": { - "termUrn": new_term, - "resourceUrn": container_urn, + "termUrn": new_term, + "resourceUrn": container_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_term_json + f"{get_frontend_url()}/api/v2/graphql", json=add_term_json ) response.raise_for_status() res_data = response.json() @@ -223,15 +222,15 @@ def test_update_container(frontend_session, ingest_cleanup_data): }""", "variables": { "input": { - "ownerUrn": new_owner, - "resourceUrn": container_urn, - "ownerEntityType": "CORP_USER" + "ownerUrn": new_owner, + "resourceUrn": container_urn, + "ownerEntityType": "CORP_USER", } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_owner_json + f"{get_frontend_url()}/api/v2/graphql", json=add_owner_json ) response.raise_for_status() res_data = response.json() @@ -248,15 +247,15 @@ def test_update_container(frontend_session, ingest_cleanup_data): }""", "variables": { "input": { - "linkUrl": new_link, - "resourceUrn": container_urn, - "label": "Label" + "linkUrl": new_link, + "resourceUrn": container_urn, + "label": "Label", } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_link_json + f"{get_frontend_url()}/api/v2/graphql", json=add_link_json ) response.raise_for_status() res_data = response.json() @@ -273,14 +272,14 @@ def test_update_container(frontend_session, ingest_cleanup_data): }""", "variables": { "input": { - "description": new_description, - "resourceUrn": container_urn, + "description": new_description, + "resourceUrn": container_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_description_json + f"{get_frontend_url()}/api/v2/graphql", json=update_description_json ) response.raise_for_status() res_data = response.json() @@ -290,7 +289,7 @@ def test_update_container(frontend_session, ingest_cleanup_data): assert res_data["data"]["updateDescription"] is True # Now fetch the container to ensure it was updated - # Get the container + # Get the container get_container_json = { "query": """query container($urn: String!) {\n container(urn: $urn) {\n @@ -327,13 +326,11 @@ def test_update_container(frontend_session, ingest_cleanup_data): }\n }\n }""", - "variables": { - "urn": container_urn - } + "variables": {"urn": container_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_container_json + f"{get_frontend_url()}/api/v2/graphql", json=get_container_json ) response.raise_for_status() res_data = response.json() @@ -348,4 +345,4 @@ def test_update_container(frontend_session, ingest_cleanup_data): assert container["ownership"]["owners"][0]["owner"]["urn"] == new_owner assert container["institutionalMemory"]["elements"][0]["url"] == new_link assert container["tags"]["tags"][0]["tag"]["urn"] == new_tag - assert container["glossaryTerms"]["terms"][0]["term"]["urn"] == new_term \ No newline at end of file + assert container["glossaryTerms"]["terms"][0]["term"]["urn"] == new_term diff --git a/smoke-test/tests/cypress/cypress.json b/smoke-test/tests/cypress/cypress.json index 2c3a885a2c5df9..9564b0077bd277 100644 --- a/smoke-test/tests/cypress/cypress.json +++ b/smoke-test/tests/cypress/cypress.json @@ -4,6 +4,7 @@ "viewportHeight": 960, "viewportWidth": 1536, "projectId": "hkrxk5", + "defaultCommandTimeout": 10000, "retries": { "runMode": 2, "openMode": 0 diff --git a/smoke-test/tests/cypress/cypress/integration/lineage/impact_analysis.js b/smoke-test/tests/cypress/cypress/integration/lineage/impact_analysis.js index 0186465472dc6a..95bd18a3afbf26 100644 --- a/smoke-test/tests/cypress/cypress/integration/lineage/impact_analysis.js +++ b/smoke-test/tests/cypress/cypress/integration/lineage/impact_analysis.js @@ -2,7 +2,8 @@ describe('mutations', () => { it('can create and add a tag to dataset and visit new tag page', () => { cy.login(); cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:kafka,SampleCypressKafkaDataset,PROD)/Lineage?is_lineage_mode=false'); - cy.contains('Impact Analysis').click({ force: true }); + // click to show more relationships now that we default to 1 degree of dependency + cy.contains('3+').click({ force: true }); // impact analysis can take a beat- don't want to time out here cy.wait(5000); diff --git a/smoke-test/tests/cypress/cypress/integration/mutations/mutations.js b/smoke-test/tests/cypress/cypress/integration/mutations/mutations.js index 6ccfca2fd92a61..bcdea9f64b0182 100644 --- a/smoke-test/tests/cypress/cypress/integration/mutations/mutations.js +++ b/smoke-test/tests/cypress/cypress/integration/mutations/mutations.js @@ -1,6 +1,7 @@ describe('mutations', () => { before(() => { // warm up elastic by issuing a `*` search + cy.login(); cy.visit('http://localhost:9002/search?query=%2A'); cy.wait(5000); }); @@ -77,6 +78,55 @@ describe('mutations', () => { cy.contains('CypressTerm').should('not.exist'); }); + it('can add and remove tags from a dataset field', () => { + cy.login(); + cy.viewport(2000, 800) + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)'); + cy.get('[data-testid="schema-field-event_name-tags"]').trigger('mouseover', {force: true}); + cy.get('[data-testid="schema-field-event_name-tags"]').within(() => cy.contains('Add Tag').click()) + + cy.focused().type('CypressTestAddTag2'); + + cy.contains('Create CypressTestAddTag2').click({force:true}); + + cy.get('textarea').type('CypressTestAddTag2 Test Description'); + + cy.contains(/Create$/).click({force:true}); + + // wait a breath for elasticsearch to index the tag being applied to the dataset- if we navigate too quick ES + // wont know and we'll see applied to 0 entities + cy.wait(2000); + + // go to tag drawer + cy.contains('CypressTestAddTag2').click(); + + cy.wait(1000); + + // Click the Tag Details to launch full profile + cy.contains('Tag Details').click(); + + cy.wait(1000); + + // title of tag page + cy.contains('CypressTestAddTag2'); + + // description of tag page + cy.contains('CypressTestAddTag2 Test Description'); + + // used by panel - click to search + cy.contains('1 Datasets').click(); + + // verify dataset shows up in search now + cy.contains('of 1 result').click(); + cy.contains('cypress_logging_events').click(); + cy.contains('CypressTestAddTag2').within(() => cy.get('span[aria-label=close]').trigger('mouseover', {force: true}).click()); + cy.contains('Yes').click(); + + cy.contains('CypressTestAddTag2').should('not.exist'); + + cy.deleteUrn('urn:li:tag:CypressTestAddTag2') + }); + it('can add and remove terms from a dataset field', () => { cy.login(); // make space for the glossary term column diff --git a/smoke-test/tests/cypress/cypress/integration/siblings/siblings.js b/smoke-test/tests/cypress/cypress/integration/siblings/siblings.js new file mode 100644 index 00000000000000..cb19284ef9cd08 --- /dev/null +++ b/smoke-test/tests/cypress/cypress/integration/siblings/siblings.js @@ -0,0 +1,148 @@ +describe('siblings', () => { + it('will merge metadata to non-primary sibling', () => { + cy.login(); + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + + // check merged platforms + cy.contains('dbt & BigQuery'); + + // check merged schema (from dbt) + cy.contains('This is a unique identifier for a customer'); + + // check merged profile (from bigquery) + cy.contains('Stats').click({ force: true }); + cy.get('[data-testid="table-stats-rowcount"]').contains("100"); + }); + + it('will merge metadata to primary sibling', () => { + cy.login(); + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + + // check merged platforms + cy.contains('dbt & BigQuery'); + + // check merged schema (from dbt) + cy.contains('This is a unique identifier for a customer'); + + // check merged profile (from bigquery) + cy.contains('Stats').click({ force: true }); + cy.get('[data-testid="table-stats-rowcount"]').contains("100"); + }); + + it('can view individual nodes', () => { + cy.login(); + + const resizeObserverLoopErrRe = /^[^(ResizeObserver loop limit exceeded)]/ + cy.on('uncaught:exception', (err) => { + /* returning false here prevents Cypress from failing the test */ + if (resizeObserverLoopErrRe.test(err.message)) { + return false + } + }) + + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + + // navigate to the bq entity + cy.get('[data-testid="compact-entity-link-urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)"]').click({ force: true }); + + // check merged platforms is not shown + cy.get('[data-testid="entity-header-test-id"]').contains('dbt & BigQuery').should('not.exist'); + cy.get('[data-testid="entity-header-test-id"]').contains('BigQuery'); + + // check dbt schema descriptions not shown + cy.contains('This is a unique identifier for a customer').should('not.exist'); + + // check merged profile still there (from bigquery) + cy.contains('Stats').click({ force: true }); + cy.get('[data-testid="table-stats-rowcount"]').contains("100"); + }); + + it('can mutate at individual node or combined node level', () => { + cy.login(); + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + + // navigate to the bq entity + cy.get('[data-testid="compact-entity-link-urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)"]').click({ force: true }); + + cy.contains('Add Term').click(); + + cy.focused().type('CypressTerm'); + + cy.get('.ant-select-item-option-content').within(() => cy.contains('CypressTerm').click({force: true})); + + cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({force: true}); + + cy.wait(2000); + + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)/?is_lineage_mode=false'); + + cy.get('a[href="/glossaryTerm/urn:li:glossaryTerm:CypressNode.CypressTerm"]').within(() => cy.get('span[aria-label=close]').click()); + cy.contains('Yes').click(); + + cy.contains('CypressTerm').should('not.exist'); + }); + + it('will combine results in search', () => { + cy.login(); + cy.visit('/search?page=1&query=%2522raw_orders%2522'); + + cy.contains('Showing 1 - 2 of 2 results'); + + cy.get('.test-search-result').should('have.length', 1); + cy.get('.test-search-result-sibling-section').should('have.length', 1); + + cy.get('.test-search-result-sibling-section').get('.test-mini-preview-class:contains(raw_orders)').should('have.length', 2); + }); + + it('will combine results in search', () => { + cy.login(); + cy.visit('/search?page=1&query=%2522raw_orders%2522'); + + cy.contains('Showing 1 - 2 of 2 results'); + + cy.get('.test-search-result').should('have.length', 1); + cy.get('.test-search-result-sibling-section').should('have.length', 1); + + cy.get('.test-search-result-sibling-section').get('.test-mini-preview-class:contains(raw_orders)').should('have.length', 2); + }); + + it('will combine results in lineage', () => { + cy.login(); + cy.visit('dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)/?is_lineage_mode=true'); + + // check the subtypes + cy.get('text:contains(Table)').should('have.length', 2); + cy.get('text:contains(Seed)').should('have.length', 1); + + // check the names + cy.get('text:contains(raw_orders)').should('have.length', 1); + cy.get('text:contains(customers)').should('have.length', 1); + // center counts twice since we secretely render two center nodes + cy.get('text:contains(stg_orders)').should('have.length', 2); + + // check the platform + cy.get('svg').get('text:contains(dbt & BigQuery)').should('have.length', 5); + }); + + it('can separate results in lineage if flag is set', () => { + cy.login(); + cy.visit('dataset/urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)/?is_lineage_mode=true'); + + cy.get('[data-testid="compress-lineage-toggle"]').click({ force: true }); + + // check the subtypes + cy.get('text:contains(View)').should('have.length', 2); + cy.get('text:contains(Table)').should('have.length', 0); + cy.get('text:contains(Seed)').should('have.length', 1); + + // check the names + cy.get('text:contains(raw_orders)').should('have.length', 1); + // center counts twice since we secretely render two center nodes, plus the downstream bigquery + cy.get('text:contains(stg_orders)').should('have.length', 3); + + // check the platform + cy.get('svg').get('text:contains(dbt & BigQuery)').should('have.length', 0); + cy.get('svg').get('text:contains(Dbt)').should('have.length', 3); + cy.get('svg').get('text:contains(Bigquery)').should('have.length', 1); + }); +}); diff --git a/smoke-test/tests/cypress/cypress_dbt_data.json b/smoke-test/tests/cypress/cypress_dbt_data.json new file mode 100644 index 00000000000000..087af0f3704c65 --- /dev/null +++ b/smoke-test/tests/cypress/cypress_dbt_data.json @@ -0,0 +1,4255 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:b5e95fce839e7d78151ed7e0a7420d84", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"bigquery\", \"instance\": \"PROD\", \"project_id\": \"cypress_project\"}, \"name\": \"cypress_project\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162350940, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:b5e95fce839e7d78151ed7e0a7420d84", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:bigquery\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162350941, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:b5e95fce839e7d78151ed7e0a7420d84", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Project\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162350942, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"bigquery\", \"instance\": \"PROD\", \"project_id\": \"cypress_project\", \"dataset_id\": \"jaffle_shop\"}, \"name\": \"jaffle_shop\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162353361, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:bigquery\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162353362, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Dataset\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162353363, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:b5e95fce839e7d78151ed7e0a7420d84\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162353364, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162353970, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162353971, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162353980, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354204, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "customers_source", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.customers_source", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "source_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "siour", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162354206, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354211, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354420, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162354421, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354427, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354667, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162354668, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354671, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354871, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162354873, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162354879, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162355105, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162355107, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162355113, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162355777, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_customers`\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "is_view": "True" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162355778, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162355783, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162355784, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_customers`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as customer_id,\\n first_name,\\n last_name\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162355785, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356113, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_orders`\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "is_view": "True" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162356115, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356123, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356124, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_orders`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as order_id,\\n user_id as customer_id,\\n order_date,\\n status\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356125, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356440, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "with source as (\n select * from `cypress_project`.`jaffle_shop`.`raw_payments`\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n --`amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "is_view": "True" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "cypress_project.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:bigquery", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Integer()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "String()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": true, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "Float()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162356441, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356445, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356446, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"with source as (\\n select * from `cypress_project`.`jaffle_shop`.`raw_payments`\\n\\n),\\n\\nrenamed as (\\n\\n select\\n id as payment_id,\\n order_id,\\n payment_method,\\n\\n --`amount` is currently stored in cents, so we convert it to dollars\\n amount / 100 as amount\\n\\n from source\\n\\n)\\n\\nselect * from renamed\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162356446, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1655162357476, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 100, \"columnCount\": 7, \"fieldProfiles\": [{\"fieldPath\": \"customer_id\", \"uniqueCount\": 100, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"61\", \"74\", \"48\", \"75\", \"87\", \"14\", \"37\", \"55\", \"49\", \"78\", \"77\", \"10\", \"15\", \"60\", \"24\", \"45\", \"62\", \"98\", \"5\", \"97\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 79, \"uniqueProportion\": 0.79, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Timothy\", \"Harry\", \"Lillian\", \"Andrea\", \"Phillip\", \"Steve\", \"Shirley\", \"Nicholas\", \"Judy\", \"Harry\", \"Anne\", \"Henry\", \"Teresa\", \"Norma\", \"David\", \"Scott\", \"Elizabeth\", \"Nicole\", \"Katherine\", \"Shirley\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 19, \"uniqueProportion\": 0.19, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"R.\", \"A.\", \"C.\", \"H.\", \"B.\", \"F.\", \"J.\", \"R.\", \"N.\", \"H.\", \"W.\", \"W.\", \"H.\", \"W.\", \"G.\", \"B.\", \"P.\", \"M.\", \"R.\", \"D.\"]}, {\"fieldPath\": \"first_order\", \"uniqueCount\": 46, \"uniqueProportion\": 0.7419354838709677, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"2018-01-01\", \"max\": \"2018-04-07\", \"sampleValues\": [\"2018-03-01\", \"2018-03-23\", \"2018-02-26\", \"2018-01-17\", \"2018-02-04\", \"2018-03-23\", \"2018-03-16\", \"2018-03-03\", \"2018-01-24\", \"2018-02-19\", \"2018-01-18\", \"2018-04-07\", \"2018-02-02\", \"2018-04-07\", \"2018-02-13\", \"2018-01-23\", \"2018-02-06\", \"2018-01-09\", \"2018-02-16\", \"2018-02-17\"]}, {\"fieldPath\": \"most_recent_order\", \"uniqueCount\": 52, \"uniqueProportion\": 0.8387096774193549, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"2018-01-09\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-01\", \"2018-03-23\", \"2018-02-26\", \"2018-01-17\", \"2018-02-04\", \"2018-03-23\", \"2018-03-16\", \"2018-03-03\", \"2018-01-24\", \"2018-02-19\", \"2018-01-18\", \"2018-04-07\", \"2018-02-02\", \"2018-04-07\", \"2018-02-13\", \"2018-01-23\", \"2018-02-06\", \"2018-01-09\", \"2018-02-16\", \"2018-02-17\"]}, {\"fieldPath\": \"number_of_orders\", \"uniqueCount\": 4, \"uniqueProportion\": 0.06451612903225806, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"1\", \"max\": \"5\", \"mean\": \"1.5967741935483863\", \"median\": \"1.0\", \"stdev\": \"0.7779687173818426\", \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"customer_lifetime_value\", \"uniqueCount\": 35, \"uniqueProportion\": 0.5645161290322581, \"nullCount\": 38, \"nullProportion\": 0.38, \"min\": \"1.0\", \"max\": \"99.0\", \"mean\": \"26.967741935483883\", \"median\": \"26.5\", \"sampleValues\": [\"2.0\", \"2.0\", \"3.0\", \"3.0\", \"3.0\", \"3.0\", \"3.0\", \"4.0\", \"8.0\", \"8.0\", \"10.0\", \"10.0\", \"12.0\", \"14.0\", \"14.0\", \"15.0\", \"15.0\", \"16.0\", \"17.0\", \"18.0\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162378272, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1655162357619, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 0, \"columnCount\": 2, \"fieldProfiles\": [{\"fieldPath\": \"source_name\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}, {\"fieldPath\": \"siour\", \"uniqueCount\": 0, \"nullCount\": 0, \"sampleValues\": []}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162378286, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1655162357642, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 99, \"columnCount\": 9, \"fieldProfiles\": [{\"fieldPath\": \"order_id\", \"uniqueCount\": 99, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"86\", \"4\", \"9\", \"44\", \"24\", \"3\", \"62\", \"95\", \"81\", \"65\", \"94\", \"42\", \"19\", \"23\", \"58\", \"59\", \"76\", \"43\", \"93\", \"15\"]}, {\"fieldPath\": \"customer_id\", \"uniqueCount\": 62, \"uniqueProportion\": 0.6262626262626263, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"48.25252525252523\", \"median\": \"50\", \"stdev\": \"27.781341350472964\", \"sampleValues\": [\"68\", \"50\", \"53\", \"66\", \"3\", \"94\", \"57\", \"27\", \"76\", \"26\", \"63\", \"92\", \"54\", \"22\", \"22\", \"30\", \"25\", \"31\", \"66\", \"25\"]}, {\"fieldPath\": \"order_date\", \"uniqueCount\": 69, \"uniqueProportion\": 0.696969696969697, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"2018-01-01\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-26\", \"2018-01-05\", \"2018-01-12\", \"2018-02-17\", \"2018-01-27\", \"2018-01-04\", \"2018-03-05\", \"2018-04-04\", \"2018-03-23\", \"2018-03-08\", \"2018-04-03\", \"2018-02-16\", \"2018-01-22\", \"2018-01-26\", \"2018-03-01\", \"2018-03-02\", \"2018-03-20\", \"2018-02-17\", \"2018-04-03\", \"2018-01-17\"]}, {\"fieldPath\": \"status\", \"uniqueCount\": 5, \"uniqueProportion\": 0.050505050505050504, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"placed\", \"completed\", \"completed\", \"completed\", \"completed\", \"completed\", \"completed\", \"placed\", \"shipped\", \"completed\", \"placed\", \"completed\", \"completed\", \"return_pending\", \"completed\", \"completed\", \"completed\", \"completed\", \"placed\", \"completed\"]}, {\"fieldPath\": \"credit_card_amount\", \"uniqueCount\": 25, \"uniqueProportion\": 0.25252525252525254, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"30.0\", \"mean\": \"8.797979797979806\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\"]}, {\"fieldPath\": \"coupon_amount\", \"uniqueCount\": 12, \"uniqueProportion\": 0.12121212121212122, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"26.0\", \"mean\": \"1.8686868686868698\", \"median\": \"0.0\", \"sampleValues\": [\"23.0\", \"25.0\", \"0.0\", \"0.0\", \"26.0\", \"1.0\", \"0.0\", \"24.0\", \"2.0\", \"0.0\", \"7.0\", \"17.0\", \"0.0\", \"0.0\", \"18.0\", \"0.0\", \"2.0\", \"0.0\", \"0.0\", \"22.0\"]}, {\"fieldPath\": \"bank_transfer_amount\", \"uniqueCount\": 19, \"uniqueProportion\": 0.1919191919191919, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"26.0\", \"mean\": \"4.151515151515151\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\"]}, {\"fieldPath\": \"gift_card_amount\", \"uniqueCount\": 11, \"uniqueProportion\": 0.1111111111111111, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"30.0\", \"mean\": \"2.07070707070707\", \"median\": \"0.0\", \"sampleValues\": [\"0.0\", \"0.0\", \"23.0\", \"11.0\", \"0.0\", \"0.0\", \"14.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"0.0\", \"6.0\", \"23.0\", \"6.0\", \"28.0\", \"0.0\", \"18.0\", \"26.0\", \"0.0\"]}, {\"fieldPath\": \"amount\", \"uniqueCount\": 32, \"uniqueProportion\": 0.32323232323232326, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0.0\", \"max\": \"58.0\", \"mean\": \"16.888888888888882\", \"median\": \"17.0\", \"sampleValues\": [\"23.0\", \"25.0\", \"23.0\", \"11.0\", \"26.0\", \"1.0\", \"14.0\", \"24.0\", \"2.0\", \"0.0\", \"7.0\", \"17.0\", \"6.0\", \"23.0\", \"24.0\", \"28.0\", \"2.0\", \"18.0\", \"26.0\", \"22.0\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162388123, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1655162357386, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 100, \"columnCount\": 3, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 100, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"20\", \"23\", \"40\", \"59\", \"74\", \"96\", \"27\", \"45\", \"53\", \"73\", \"87\", \"4\", \"41\", \"46\", \"48\", \"64\", \"71\", \"86\", \"12\", \"82\"]}, {\"fieldPath\": \"first_name\", \"uniqueCount\": 79, \"uniqueProportion\": 0.79, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Anna\", \"Mildred\", \"Maria\", \"Adam\", \"Harry\", \"Jacqueline\", \"Benjamin\", \"Scott\", \"Anne\", \"Alan\", \"Phillip\", \"Jimmy\", \"Gloria\", \"Norma\", \"Lillian\", \"David\", \"Gerald\", \"Jason\", \"Amy\", \"Arthur\"]}, {\"fieldPath\": \"last_name\", \"uniqueCount\": 19, \"uniqueProportion\": 0.19, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"A.\", \"A.\", \"A.\", \"A.\", \"A.\", \"A.\", \"B.\", \"B.\", \"B.\", \"B.\", \"B.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"C.\", \"D.\", \"D.\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162388138, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1655162357622, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 99, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 99, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"84\", \"86\", \"87\", \"89\", \"91\", \"92\", \"93\", \"94\", \"95\", \"96\", \"97\", \"98\", \"99\", \"71\", \"72\", \"74\", \"77\", \"78\", \"79\", \"80\"]}, {\"fieldPath\": \"user_id\", \"uniqueCount\": 62, \"uniqueProportion\": 0.6262626262626263, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"48.252525252525245\", \"median\": \"50\", \"stdev\": \"27.781341350472957\", \"sampleValues\": [\"70\", \"68\", \"46\", \"21\", \"47\", \"84\", \"66\", \"63\", \"27\", \"90\", \"89\", \"41\", \"85\", \"42\", \"30\", \"9\", \"35\", \"90\", \"52\", \"11\"]}, {\"fieldPath\": \"order_date\", \"uniqueCount\": 69, \"uniqueProportion\": 0.696969696969697, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"2018-01-01\", \"max\": \"2018-04-09\", \"sampleValues\": [\"2018-03-26\", \"2018-03-26\", \"2018-03-27\", \"2018-03-28\", \"2018-03-31\", \"2018-04-02\", \"2018-04-03\", \"2018-04-03\", \"2018-04-04\", \"2018-04-06\", \"2018-04-07\", \"2018-04-07\", \"2018-04-09\", \"2018-03-12\", \"2018-03-14\", \"2018-03-17\", \"2018-03-21\", \"2018-03-23\", \"2018-03-23\", \"2018-03-23\"]}, {\"fieldPath\": \"status\", \"uniqueCount\": 5, \"uniqueProportion\": 0.050505050505050504, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"placed\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\", \"shipped\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162388145, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1655162357609, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 113, \"columnCount\": 4, \"fieldProfiles\": [{\"fieldPath\": \"id\", \"uniqueCount\": 113, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"66\", \"27\", \"30\", \"109\", \"3\", \"17\", \"47\", \"108\", \"4\", \"86\", \"93\", \"106\", \"98\", \"48\", \"107\", \"92\", \"49\", \"22\", \"67\", \"71\"]}, {\"fieldPath\": \"order_id\", \"uniqueCount\": 99, \"uniqueProportion\": 0.8761061946902655, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"1\", \"max\": \"99\", \"mean\": \"50.03539823008851\", \"median\": \"51\", \"stdev\": \"28.54317819535489\", \"sampleValues\": [\"58\", \"24\", \"25\", \"95\", \"3\", \"15\", \"42\", \"94\", \"4\", \"76\", \"81\", \"92\", \"86\", \"43\", \"93\", \"80\", \"44\", \"19\", \"58\", \"62\"]}, {\"fieldPath\": \"payment_method\", \"uniqueCount\": 4, \"uniqueProportion\": 0.035398230088495575, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"coupon\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\", \"gift_card\"]}, {\"fieldPath\": \"amount\", \"uniqueCount\": 30, \"uniqueProportion\": 0.26548672566371684, \"nullCount\": 0, \"nullProportion\": 0.0, \"min\": \"0\", \"max\": \"3000\", \"mean\": \"1479.6460176991145\", \"median\": \"1500\", \"stdev\": \"919.836873351873\", \"sampleValues\": [\"1800\", \"2600\", \"1600\", \"2400\", \"100\", \"2200\", \"1700\", \"700\", \"2500\", \"200\", \"200\", \"200\", \"2300\", \"1800\", \"2600\", \"300\", \"1100\", \"600\", \"600\", \"1400\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162388150, + "runId": "bigquery-2022_06_13-16_18_59", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322398, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/orders.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for an order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "Foreign key to the customers table", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) that the order was placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by credit card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by coupon", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by bank transfer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by gift card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "Total amount (AUD) of the order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by 1\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n left join order_payments using (order_id)\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322399, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322404, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_customers.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322405, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322409, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_payments.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n --`amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322410, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322414, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_orders.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322415, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322419, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transformers/transformers_customers.sql", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "transformers_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.transformers_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322420, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322423, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "data/raw_customers.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322423, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322427, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "data/raw_orders.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322427, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322431, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "data/raw_payments.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322432, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322509, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/customers.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for a customer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's first name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's last name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's first order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's most recent order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": false, + "description": "Count of the number of orders a customer has placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.transformers_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\nephemeral_customers as (\n\n select * from {{ ref('transformers_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\nsource_customers as (\n\n select * from {{ source('jaffle_shop', 'customers_source') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by 1\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders using (order_id)\n\n group by 1\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders using (customer_id)\n\n left join customer_payments using (customer_id)\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322510, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1655162322523, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers_source,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/schema.yml", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "manifest_version": "1.0.4", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.0.4" + }, + "externalUrl": null, + "name": "customers_source", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.jaffle_shop.jaffle_shop.customers_source", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers_source,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322524, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322527, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322528, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322529, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322530, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322531, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322532, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322533, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1655162322536, + "runId": "dbt-2022_06_13-16_18_42", + "registryName": null, + "registryVersion": null, + "properties": null + } + } +] diff --git a/smoke-test/tests/cypress/example_siblings_to_datahub_rest.yml b/smoke-test/tests/cypress/example_siblings_to_datahub_rest.yml new file mode 100644 index 00000000000000..89b259a9ea3c3c --- /dev/null +++ b/smoke-test/tests/cypress/example_siblings_to_datahub_rest.yml @@ -0,0 +1,11 @@ +# see https://datahubproject.io/docs/generated/ingestion/sources/file for complete documentation +source: + type: "file" + config: + filename: "./cypress_dbt_data.json" + +# see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" diff --git a/smoke-test/tests/cypress/integration_test.py b/smoke-test/tests/cypress/integration_test.py index 2714fbae3bf63b..a41c5f7659399f 100644 --- a/smoke-test/tests/cypress/integration_test.py +++ b/smoke-test/tests/cypress/integration_test.py @@ -10,10 +10,12 @@ def ingest_cleanup_data(): print("ingesting test data") ingest_file_via_rest("tests/cypress/data.json") + ingest_file_via_rest("tests/cypress/cypress_dbt_data.json") ingest_file_via_rest("tests/cypress/schema-blame-data.json") yield print("removing test data") delete_urns_from_file("tests/cypress/data.json") + delete_urns_from_file("tests/cypress/cypress_dbt_data.json") delete_urns_from_file("tests/cypress/schema-blame-data.json") @@ -22,10 +24,12 @@ def test_run_cypress(frontend_session, wait_for_healthchecks): record_key = os.getenv("CYPRESS_RECORD_KEY") if record_key: print('Running Cypress tests with recording') - command = f"npx cypress run --record" + command = "NO_COLOR=1 npx cypress run --record" else: print('Running Cypress tests without recording') - command = f"npx cypress run" + command = "NO_COLOR=1 npx cypress run" + # Add --headed --spec '**/mutations/mutations.js' (change spec name) + # in case you want to see the browser for debugging proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd="tests/cypress") stdout = proc.stdout.read() stderr = proc.stderr.read() diff --git a/smoke-test/tests/delete/delete_test.py b/smoke-test/tests/delete/delete_test.py index a5a855daa23311..dc04542cb33baf 100644 --- a/smoke-test/tests/delete/delete_test.py +++ b/smoke-test/tests/delete/delete_test.py @@ -1,11 +1,27 @@ +import os import json import pytest from time import sleep -from datahub.cli import delete_cli, ingest_cli -from datahub.cli.cli_utils import guess_entity_type, post_entity, get_aspects_for_entity +from datahub.cli.cli_utils import get_aspects_for_entity from datahub.cli.ingest_cli import get_session_and_host -from datahub.cli.delete_cli import guess_entity_type, delete_one_urn_cmd, delete_references -from tests.utils import ingest_file_via_rest, delete_urns_from_file +from datahub.cli.delete_cli import delete_references +from tests.utils import ingest_file_via_rest, wait_for_healthcheck_util + +# Disable telemetry +os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false" + + +@pytest.fixture(scope="session") +def wait_for_healthchecks(): + wait_for_healthcheck_util() + yield + + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + @pytest.fixture(autouse=True) def test_setup(): @@ -24,7 +40,7 @@ def test_setup(): ingested_dataset_run_id = ingest_file_via_rest("tests/delete/cli_test_data.json").config.run_id - sleep(2) + sleep(3) assert "browsePaths" in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["browsePaths"], typed=False) @@ -32,13 +48,14 @@ def test_setup(): rollback_url = f"{gms_host}/runs?action=rollback" session.post(rollback_url, data=json.dumps({"runId": ingested_dataset_run_id, "dryRun": False, "hardDelete": True, "safe": False})) - sleep(2) + sleep(3) assert "browsePaths" not in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["browsePaths"], typed=False) assert "editableDatasetProperties" not in get_aspects_for_entity(entity_urn=dataset_urn, aspects=["editableDatasetProperties"], typed=False) + @pytest.mark.dependency() -def test_delete_reference(): +def test_delete_reference(depends=["test_healthchecks"]): platform = "urn:li:dataPlatform:kafka" dataset_name = "test-delete" @@ -58,7 +75,7 @@ def test_delete_reference(): # Delete references to the tag delete_references(tag_urn, dry_run=False, cached_session_host=(session, gms_host)) - sleep(2) + sleep(3) # Validate that references no longer exist references_count, related_aspects = delete_references(tag_urn, dry_run=True, cached_session_host=(session, gms_host)) diff --git a/smoke-test/tests/deprecation/deprecation_test.py b/smoke-test/tests/deprecation/deprecation_test.py index 1173ca8d6327de..ffa2c131962071 100644 --- a/smoke-test/tests/deprecation/deprecation_test.py +++ b/smoke-test/tests/deprecation/deprecation_test.py @@ -1,9 +1,6 @@ import pytest -import time -from tests.utils import FRONTEND_ENDPOINT -from tests.utils import GMS_ENDPOINT -from tests.utils import ingest_file_via_rest -from tests.utils import delete_urns_from_file +from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest + @pytest.fixture(scope="module", autouse=True) def ingest_cleanup_data(request): @@ -13,14 +10,18 @@ def ingest_cleanup_data(request): print("removing deprecation test data") delete_urns_from_file("tests/deprecation/data.json") + @pytest.mark.dependency() def test_healthchecks(wait_for_healthchecks): # Call to wait_for_healthchecks fixture will do the actual functionality. pass + @pytest.mark.dependency(depends=["test_healthchecks"]) def test_update_deprecation_all_fields(frontend_session): - dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" + dataset_urn = ( + "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" + ) dataset_json = { "query": """query getDataset($urn: String!) {\n @@ -33,14 +34,12 @@ def test_update_deprecation_all_fields(frontend_session): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } # Fetch tags response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -56,16 +55,16 @@ def test_update_deprecation_all_fields(frontend_session): }""", "variables": { "input": { - "urn": dataset_urn, - "deprecated": True, - "note": "My test note", - "decommissionTime": 0 + "urn": dataset_urn, + "deprecated": True, + "note": "My test note", + "decommissionTime": 0, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_deprecation_json + f"{get_frontend_url()}/api/v2/graphql", json=update_deprecation_json ) response.raise_for_status() res_data = response.json() @@ -76,7 +75,7 @@ def test_update_deprecation_all_fields(frontend_session): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -85,30 +84,30 @@ def test_update_deprecation_all_fields(frontend_session): assert res_data["data"] assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]["deprecation"] == { - 'deprecated': True, - 'decommissionTime': 0, - 'note': 'My test note', - 'actor': 'urn:li:corpuser:datahub' + "deprecated": True, + "decommissionTime": 0, + "note": "My test note", + "actor": "urn:li:corpuser:datahub", } -@pytest.mark.dependency(depends=["test_healthchecks", "test_update_deprecation_all_fields"]) + +@pytest.mark.dependency( + depends=["test_healthchecks", "test_update_deprecation_all_fields"] +) def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data): - dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" + dataset_urn = ( + "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" + ) update_deprecation_json = { "query": """mutation updateDeprecation($input: UpdateDeprecationInput!) {\n updateDeprecation(input: $input) }""", - "variables": { - "input": { - "urn": dataset_urn, - "deprecated": False - } - } + "variables": {"input": {"urn": dataset_urn, "deprecated": False}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_deprecation_json + f"{get_frontend_url()}/api/v2/graphql", json=update_deprecation_json ) response.raise_for_status() res_data = response.json() @@ -129,13 +128,11 @@ def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -144,8 +141,8 @@ def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data assert res_data["data"] assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]["deprecation"] == { - 'deprecated': False, - 'note': '', - 'actor': 'urn:li:corpuser:datahub', - 'decommissionTime': None - } \ No newline at end of file + "deprecated": False, + "note": "", + "actor": "urn:li:corpuser:datahub", + "decommissionTime": None, + } diff --git a/smoke-test/tests/domains/domains_test.py b/smoke-test/tests/domains/domains_test.py index 11b41cb4240a88..a53ef71c9e4143 100644 --- a/smoke-test/tests/domains/domains_test.py +++ b/smoke-test/tests/domains/domains_test.py @@ -1,9 +1,17 @@ -import pytest import time -from tests.utils import FRONTEND_ENDPOINT -from tests.utils import GMS_ENDPOINT -from tests.utils import ingest_file_via_rest -from tests.utils import delete_urns_from_file + +import pytest +import tenacity +from tests.utils import ( + delete_urns_from_file, + get_frontend_url, + get_gms_url, + ingest_file_via_rest, + get_sleep_info, +) + +sleep_sec, sleep_times = get_sleep_info() + @pytest.fixture(scope="module", autouse=False) def ingest_cleanup_data(request): @@ -13,11 +21,37 @@ def ingest_cleanup_data(request): print("removing domains test data") delete_urns_from_file("tests/domains/data.json") + @pytest.mark.dependency() def test_healthchecks(wait_for_healthchecks): # Call to wait_for_healthchecks fixture will do the actual functionality. pass + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_more_domains(frontend_session, list_domains_json, before_count): + time.sleep(2) + + # Get new count of Domains + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=list_domains_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["listDomains"]["total"] is not None + assert "errors" not in res_data + + # Assert that there are more domains now. + after_count = res_data["data"]["listDomains"]["total"] + print(f"after_count is {after_count}") + assert after_count == before_count + 1 + + @pytest.mark.dependency(depends=["test_healthchecks"]) def test_create_list_get_domain(frontend_session): @@ -36,16 +70,11 @@ def test_create_list_get_domain(frontend_session): }\n }\n }""", - "variables": { - "input": { - "start": "0", - "count": "20" - } - } + "variables": {"input": {"start": "0", "count": "20"}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=list_domains_json + f"{get_frontend_url()}/api/v2/graphql", json=list_domains_json ) response.raise_for_status() res_data = response.json() @@ -54,9 +83,10 @@ def test_create_list_get_domain(frontend_session): assert res_data["data"] assert res_data["data"]["listDomains"]["total"] is not None assert "errors" not in res_data + print(f"domains resp is {res_data}") before_count = res_data["data"]["listDomains"]["total"] - print(before_count) + print(f"before_count is {before_count}") domain_id = "test id" domain_name = "test name" @@ -68,16 +98,16 @@ def test_create_list_get_domain(frontend_session): createDomain(input: $input) }""", "variables": { - "input": { - "id": domain_id, - "name": domain_name, - "description": domain_description - } - } + "input": { + "id": domain_id, + "name": domain_name, + "description": domain_description, + } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=create_domain_json + f"{get_frontend_url()}/api/v2/graphql", json=create_domain_json ) response.raise_for_status() res_data = response.json() @@ -89,26 +119,11 @@ def test_create_list_get_domain(frontend_session): domain_urn = res_data["data"]["createDomain"] - # Sleep for eventual consistency (not ideal) - time.sleep(2) - - # Get new count of Domains - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=list_domains_json + _ensure_more_domains( + frontend_session=frontend_session, + list_domains_json=list_domains_json, + before_count=before_count, ) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["listDomains"]["total"] is not None - assert "errors" not in res_data - - # Assert that there are more domains now. - after_count = res_data["data"]["listDomains"]["total"] - print(after_count) - assert after_count == before_count + 1 - # Get the domain value back get_domain_json = { @@ -122,13 +137,11 @@ def test_create_list_get_domain(frontend_session): }\n }\n }""", - "variables": { - "urn": domain_urn - } + "variables": {"urn": domain_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_domain_json + f"{get_frontend_url()}/api/v2/graphql", json=get_domain_json ) response.raise_for_status() res_data = response.json() @@ -139,18 +152,16 @@ def test_create_list_get_domain(frontend_session): assert "errors" not in res_data domain = res_data["data"]["domain"] - assert domain["urn"] == f'urn:li:domain:{domain_id}' + assert domain["urn"] == f"urn:li:domain:{domain_id}" assert domain["id"] == domain_id assert domain["properties"]["name"] == domain_name assert domain["properties"]["description"] == domain_description - delete_json = { - "urn": domain_urn - } + delete_json = {"urn": domain_urn} # Cleanup: Delete the domain response = frontend_session.post( - f"{GMS_ENDPOINT}/entities?action=delete", json=delete_json + f"{get_gms_url()}/entities?action=delete", json=delete_json ) response.raise_for_status() @@ -160,20 +171,20 @@ def test_create_list_get_domain(frontend_session): def test_set_unset_domain(frontend_session, ingest_cleanup_data): # Set and Unset a Domain for a dataset. Note that this doesn't test for adding domains to charts, dashboards, charts, & jobs. - dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" + dataset_urn = ( + "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" + ) domain_urn = "urn:li:domain:engineering" # First unset to be sure. unset_domain_json = { "query": """mutation unsetDomain($entityUrn: String!) {\n unsetDomain(entityUrn: $entityUrn)}""", - "variables": { - "entityUrn": dataset_urn - } + "variables": {"entityUrn": dataset_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=unset_domain_json + f"{get_frontend_url()}/api/v2/graphql", json=unset_domain_json ) response.raise_for_status() res_data = response.json() @@ -187,14 +198,11 @@ def test_set_unset_domain(frontend_session, ingest_cleanup_data): set_domain_json = { "query": """mutation setDomain($entityUrn: String!, $domainUrn: String!) {\n setDomain(entityUrn: $entityUrn, domainUrn: $domainUrn)}""", - "variables": { - "entityUrn": dataset_urn, - "domainUrn": domain_urn - } + "variables": {"entityUrn": dataset_urn, "domainUrn": domain_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=set_domain_json + f"{get_frontend_url()}/api/v2/graphql", json=set_domain_json ) response.raise_for_status() res_data = response.json() @@ -204,30 +212,30 @@ def test_set_unset_domain(frontend_session, ingest_cleanup_data): assert res_data["data"]["setDomain"] is True assert "errors" not in res_data - # Now, fetch the dataset's domain and confirm it was set.GMS_ENDPOINT + # Now, fetch the dataset's domain and confirm it was set. get_dataset_json = { "query": """query dataset($urn: String!) {\n dataset(urn: $urn) {\n urn\n domain {\n - urn\n - properties{\n - name\n + domain {\n + urn\n + properties{\n + name\n + }\n }\n }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=get_dataset_json ) response.raise_for_status() res_data = response.json() assert res_data - assert res_data["data"]["dataset"]["domain"]["urn"] == domain_urn - assert res_data["data"]["dataset"]["domain"]["properties"]["name"] == "Engineering" + assert res_data["data"]["dataset"]["domain"]["domain"]["urn"] == domain_urn + assert res_data["data"]["dataset"]["domain"]["domain"]["properties"]["name"] == "Engineering" diff --git a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py index 8a149eccaaea24..4a3d30d2f4d0b0 100644 --- a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py +++ b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py @@ -1,76 +1,70 @@ -import pytest +import json import time -import requests -from tests.utils import FRONTEND_ENDPOINT -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_create_list_get_remove_secret(frontend_session): +import pytest +import tenacity - # Get count of existing secrets - json = { - "query": """query listSecrets($input: ListSecretsInput!) {\n - listSecrets(input: $input) {\n +from tests.utils import (get_frontend_url, get_sleep_info, + wait_for_healthcheck_util) + +sleep_sec, sleep_times = get_sleep_info() + + +@pytest.fixture(scope="session") +def wait_for_healthchecks(): + wait_for_healthcheck_util() + yield + + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + + +def _get_ingestionSources(frontend_session): + json_q = { + "query": """query listIngestionSources($input: ListIngestionSourcesInput!) {\n + listIngestionSources(input: $input) {\n start\n count\n total\n - secrets {\n + ingestionSources {\n urn\n - name\n }\n }\n }""", - "variables": { - "input": { - "start": "0", - "count": "20" - } - } + "variables": {"input": {"start": "0", "count": "20"}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["listSecrets"]["total"] is not None + assert res_data["data"]["listIngestionSources"]["total"] is not None assert "errors" not in res_data + return res_data - before_count = res_data["data"]["listSecrets"]["total"] - # Create new secret - json = { - "query": """mutation createSecret($input: CreateSecretInput!) {\n - createSecret(input: $input) - }""", - "variables": { - "input": { - "name": "SMOKE_TEST", - "value": "mytestvalue" - } - } - } - - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["createSecret"] is not None - assert "errors" not in res_data - - secret_urn = res_data["data"]["createSecret"] +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_ingestion_source_count(frontend_session, expected_count): + res_data = _get_ingestionSources(frontend_session) + after_count = res_data["data"]["listIngestionSources"]["total"] + assert after_count == expected_count + return after_count - # Sleep for eventual consistency (not ideal) - time.sleep(2) - # Get new count of secrets - json = { +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_secret_increased(frontend_session, before_count): + json_q = { "query": """query listSecrets($input: ListSecretsInput!) {\n listSecrets(input: $input) {\n start\n @@ -82,16 +76,11 @@ def test_create_list_get_remove_secret(frontend_session): }\n }\n }""", - "variables": { - "input": { - "start": "0", - "count": "20" - } - } + "variables": {"input": {"start": "0", "count": "20"}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() @@ -105,203 +94,268 @@ def test_create_list_get_remove_secret(frontend_session): after_count = res_data["data"]["listSecrets"]["total"] assert after_count == before_count + 1 + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_secret_not_present(frontend_session): # Get the secret value back - json = { + json_q = { "query": """query getSecretValues($input: GetSecretValuesInput!) {\n getSecretValues(input: $input) {\n name\n value\n }\n }""", - "variables": { - "input": { - "secrets": ["SMOKE_TEST"] - } - } + "variables": {"input": {"secrets": ["SMOKE_TEST"]}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() - print(res_data) assert res_data assert res_data["data"] assert res_data["data"]["getSecretValues"] is not None assert "errors" not in res_data secret_values = res_data["data"]["getSecretValues"] - secret_value = [x for x in secret_values if x["name"] == "SMOKE_TEST"][0] - assert secret_value["value"] == "mytestvalue" + secret_value_arr = [x for x in secret_values if x["name"] == "SMOKE_TEST"] + assert len(secret_value_arr) == 0 - # Now cleanup and remove the secret - json = { - "query": """mutation deleteSecret($urn: String!) {\n - deleteSecret(urn: $urn) + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_ingestion_source_present( + frontend_session, ingestion_source_urn, num_execs=None +): + json_q = { + "query": """query ingestionSource($urn: String!) {\n + ingestionSource(urn: $urn) {\n + executions(start: 0, count: 1) {\n + start\n + count\n + total\n + executionRequests {\n + urn\n + }\n + }\n + }\n }""", - "variables": { - "urn": secret_urn - } + "variables": {"urn": ingestion_source_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["deleteSecret"] is not None + assert res_data["data"]["ingestionSource"] is not None assert "errors" not in res_data - # Re-fetch the secret values and see that they are not there. - time.sleep(2) + if num_execs is not None: + ingestion_source = res_data["data"]["ingestionSource"] + assert ingestion_source["executions"]["total"] == num_execs - # Get the secret value back - json = { - "query": """query getSecretValues($input: GetSecretValuesInput!) {\n - getSecretValues(input: $input) {\n - name\n - value\n + return res_data + + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_execution_request_present(frontend_session, execution_request_urn): + json_q = { + "query": """query executionRequest($urn: String!) {\n + executionRequest(urn: $urn) {\n + urn\n + input {\n + task\n + arguments {\n + key\n + value\n + }\n + }\n + result {\n + status\n + startTimeMs\n + durationMs\n + }\n }\n }""", - "variables": { - "input": { - "secrets": ["SMOKE_TEST"] - } - } + "variables": {"urn": execution_request_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["getSecretValues"] is not None + assert res_data["data"]["executionRequest"] is not None assert "errors" not in res_data + return res_data - secret_values = res_data["data"]["getSecretValues"] - secret_value_arr = [x for x in secret_values if x["name"] == "SMOKE_TEST"] - assert len(secret_value_arr) == 0 -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_create_list_get_remove_ingestion_source(frontend_session): +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_create_list_get_remove_secret(frontend_session): - # Get count of existing ingestion sources - json = { - "query": """query listIngestionSources($input: ListIngestionSourcesInput!) {\n - listIngestionSources(input: $input) {\n + # Get count of existing secrets + json_q = { + "query": """query listSecrets($input: ListSecretsInput!) {\n + listSecrets(input: $input) {\n start\n count\n total\n - ingestionSources {\n + secrets {\n urn\n + name\n }\n }\n }""", - "variables": { - "input": { - "start": "0", - "count": "20" - } - } + "variables": {"input": {"start": "0", "count": "20"}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["listIngestionSources"]["total"] is not None + assert res_data["data"]["listSecrets"]["total"] is not None assert "errors" not in res_data - before_count = res_data["data"]["listIngestionSources"]["total"] + before_count = res_data["data"]["listSecrets"]["total"] - # Create new ingestion source - json = { - "query": """mutation createIngestionSource($input: UpdateIngestionSourceInput!) {\n - createIngestionSource(input: $input) + # Create new secret + json_q = { + "query": """mutation createSecret($input: CreateSecretInput!) {\n + createSecret(input: $input) }""", - "variables": { - "input": { - "name": "My Test Ingestion Source", - "type": "mysql", - "description": "My ingestion source description", - "schedule": { - "interval": "* * * * *", - "timezone": "UTC" - }, - "config": { - "recipe": "MY_TEST_RECIPE", - "version": "0.8.18", - "executorId": "mytestexecutor" - } - } - } + "variables": {"input": {"name": "SMOKE_TEST", "value": "mytestvalue"}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["createIngestionSource"] is not None + assert res_data["data"]["createSecret"] is not None assert "errors" not in res_data - ingestion_source_urn = res_data["data"]["createIngestionSource"] + secret_urn = res_data["data"]["createSecret"] - # Sleep for eventual consistency (not ideal) - time.sleep(2) + # Get new count of secrets + _ensure_secret_increased(frontend_session, before_count) - # Get new count of ingestion sources - json = { - "query": """query listIngestionSources($input: ListIngestionSourcesInput!) {\n - listIngestionSources(input: $input) {\n - start\n - count\n - total\n - ingestionSources {\n - urn\n - }\n + # Get the secret value back + json_q = { + "query": """query getSecretValues($input: GetSecretValuesInput!) {\n + getSecretValues(input: $input) {\n + name\n + value\n }\n }""", + "variables": {"input": {"secrets": ["SMOKE_TEST"]}}, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=json_q + ) + response.raise_for_status() + res_data = response.json() + + print(res_data) + assert res_data + assert res_data["data"] + assert res_data["data"]["getSecretValues"] is not None + assert "errors" not in res_data + + secret_values = res_data["data"]["getSecretValues"] + secret_value = [x for x in secret_values if x["name"] == "SMOKE_TEST"][0] + assert secret_value["value"] == "mytestvalue" + + # Now cleanup and remove the secret + json_q = { + "query": """mutation deleteSecret($urn: String!) {\n + deleteSecret(urn: $urn) + }""", + "variables": {"urn": secret_urn}, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=json_q + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["deleteSecret"] is not None + assert "errors" not in res_data + + # Re-fetch the secret values and see that they are not there. + _ensure_secret_not_present(frontend_session) + + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_create_list_get_remove_ingestion_source(frontend_session): + + # Get count of existing ingestion sources + res_data = _get_ingestionSources(frontend_session) + + before_count = res_data["data"]["listIngestionSources"]["total"] + + # Create new ingestion source + json_q = { + "query": """mutation createIngestionSource($input: UpdateIngestionSourceInput!) {\n + createIngestionSource(input: $input) + }""", "variables": { - "input": { - "start": "0", - "count": "20" - } - } + "input": { + "name": "My Test Ingestion Source", + "type": "mysql", + "description": "My ingestion source description", + "schedule": {"interval": "*/5 * * * *", "timezone": "UTC"}, + "config": { + "recipe": '{"source":{"type":"mysql","config":{"include_tables":true,"database":null,"password":"${MYSQL_PASSWORD}","profiling":{"enabled":false},"host_port":null,"include_views":true,"username":"${MYSQL_USERNAME}"}},"pipeline_name":"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927"}', + "version": "0.8.18", + "executorId": "mytestexecutor", + }, + } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["listIngestionSources"]["total"] is not None + assert res_data["data"]["createIngestionSource"] is not None assert "errors" not in res_data + ingestion_source_urn = res_data["data"]["createIngestionSource"] + # Assert that there are more ingestion sources now. - after_count = res_data["data"]["listIngestionSources"]["total"] - assert after_count == before_count + 1 + after_count = _ensure_ingestion_source_count(frontend_session, before_count + 1) # Get the ingestion source back - json = { + json_q = { "query": """query ingestionSource($urn: String!) {\n ingestionSource(urn: $urn) {\n urn\n @@ -318,13 +372,11 @@ def test_create_list_get_remove_ingestion_source(frontend_session): }\n }\n }""", - "variables": { - "urn": ingestion_source_urn - } + "variables": {"urn": ingestion_source_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() @@ -338,24 +390,25 @@ def test_create_list_get_remove_ingestion_source(frontend_session): assert ingestion_source["urn"] == ingestion_source_urn assert ingestion_source["type"] == "mysql" assert ingestion_source["name"] == "My Test Ingestion Source" - assert ingestion_source["schedule"]["interval"] == "* * * * *" + assert ingestion_source["schedule"]["interval"] == "*/5 * * * *" assert ingestion_source["schedule"]["timezone"] == "UTC" - assert ingestion_source["config"]["recipe"] == "MY_TEST_RECIPE" + assert ( + ingestion_source["config"]["recipe"] + == '{"source":{"type":"mysql","config":{"include_tables":true,"database":null,"password":"${MYSQL_PASSWORD}","profiling":{"enabled":false},"host_port":null,"include_views":true,"username":"${MYSQL_USERNAME}"}},"pipeline_name":"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927"}' + ) assert ingestion_source["config"]["executorId"] == "mytestexecutor" assert ingestion_source["config"]["version"] == "0.8.18" # Now cleanup and remove the ingestion source - json = { + json_q = { "query": """mutation deleteIngestionSource($urn: String!) {\n deleteIngestionSource(urn: $urn) }""", - "variables": { - "urn": ingestion_source_urn - } + "variables": {"urn": ingestion_source_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() @@ -366,70 +419,39 @@ def test_create_list_get_remove_ingestion_source(frontend_session): assert res_data["data"]["deleteIngestionSource"] is not None assert "errors" not in res_data - # Re-fetch the ingestion sources and see that they are not there. - time.sleep(2) - # Ensure the ingestion source has been removed. - json = { - "query": """query listIngestionSources($input: ListIngestionSourcesInput!) {\n - listIngestionSources(input: $input) {\n - start\n - count\n - total\n - ingestionSources {\n - urn\n - }\n - }\n - }""", - "variables": { - "input": { - "start": "0", - "count": "20" - } - } - } + _ensure_ingestion_source_count(frontend_session, after_count - 1) - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["listIngestionSources"]["total"] is not None - assert "errors" not in res_data - final_count = res_data["data"]["listIngestionSources"]["total"] - assert final_count == after_count - 1 - -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion", "test_create_list_get_remove_ingestion_source"]) +@pytest.mark.dependency( + depends=[ + "test_healthchecks", + "test_create_list_get_remove_ingestion_source", + ] +) def test_create_list_get_ingestion_execution_request(frontend_session): # Create new ingestion source - json = { + json_q = { "query": """mutation createIngestionSource($input: UpdateIngestionSourceInput!) {\n createIngestionSource(input: $input) }""", "variables": { - "input": { - "name": "My Test Ingestion Source", - "type": "mysql", - "description": "My ingestion source description", - "schedule": { - "interval": "* * * * *", - "timezone": "UTC" - }, - "config": { - "recipe": "MY_TEST_RECIPE", - "version": "0.8.18", - "executorId": "mytestexecutor" - } - } - } + "input": { + "name": "My Test Ingestion Source", + "type": "mysql", + "description": "My ingestion source description", + "schedule": {"interval": "*/5 * * * *", "timezone": "UTC"}, + "config": { + "recipe": '{"source":{"type":"mysql","config":{"include_tables":true,"database":null,"password":"${MYSQL_PASSWORD}","profiling":{"enabled":false},"host_port":null,"include_views":true,"username":"${MYSQL_USERNAME}"}},"pipeline_name":"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927"}', + "version": "0.8.18", + "executorId": "mytestexecutor", + }, + } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() @@ -442,101 +464,43 @@ def test_create_list_get_ingestion_execution_request(frontend_session): ingestion_source_urn = res_data["data"]["createIngestionSource"] # Create a request to execute the ingestion source - json = { + json_q = { "query": """mutation createIngestionExecutionRequest($input: CreateIngestionExecutionRequestInput!) {\n createIngestionExecutionRequest(input: $input) }""", - "variables": { - "input": { - "ingestionSourceUrn": ingestion_source_urn - } - } + "variables": {"input": {"ingestionSourceUrn": ingestion_source_urn}}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() assert res_data assert res_data["data"] - assert res_data["data"]["createIngestionExecutionRequest"] is not None + assert ( + res_data["data"]["createIngestionExecutionRequest"] is not None + ), f"res_data was {res_data}" assert "errors" not in res_data execution_request_urn = res_data["data"]["createIngestionExecutionRequest"] - # Wait for eventual consistency. - time.sleep(2) - - # Get the ingestion source executions - json = { - "query": """query ingestionSource($urn: String!) {\n - ingestionSource(urn: $urn) {\n - executions(start: 0, count: 1) {\n - start\n - count\n - total\n - executionRequests {\n - urn\n - }\n - }\n - }\n - }""", - "variables": { - "urn": ingestion_source_urn - } - } - - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + res_data = _ensure_ingestion_source_present( + frontend_session, ingestion_source_urn, 1 ) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["ingestionSource"] is not None - assert "errors" not in res_data ingestion_source = res_data["data"]["ingestionSource"] - assert ingestion_source["executions"]["total"] == 1 - assert ingestion_source["executions"]["executionRequests"][0]["urn"] == execution_request_urn - - # Get the ingestion request back via direct lookup - json = { - "query": """query executionRequest($urn: String!) {\n - executionRequest(urn: $urn) {\n - urn\n - input {\n - task\n - arguments {\n - key\n - value\n - }\n - }\n - result {\n - status\n - startTimeMs\n - durationMs\n - }\n - }\n - }""", - "variables": { - "urn": execution_request_urn - } - } - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + assert ( + ingestion_source["executions"]["executionRequests"][0]["urn"] + == execution_request_urn ) - response.raise_for_status() - res_data = response.json() - assert res_data - assert res_data["data"] - assert res_data["data"]["executionRequest"] is not None - assert "errors" not in res_data + # Get the ingestion request back via direct lookup + res_data = _ensure_execution_request_present( + frontend_session, execution_request_urn + ) execution_request = res_data["data"]["executionRequest"] assert execution_request["urn"] == execution_request_urn @@ -544,26 +508,29 @@ def test_create_list_get_ingestion_execution_request(frontend_session): # Verify input assert execution_request["input"]["task"] == "RUN_INGEST" assert len(execution_request["input"]["arguments"]) == 2 - assert execution_request["input"]["arguments"][0]["key"] == 'recipe' - assert execution_request["input"]["arguments"][0]["value"] == 'MY_TEST_RECIPE' - assert execution_request["input"]["arguments"][1]["key"] == 'version' - assert execution_request["input"]["arguments"][1]["value"] == '0.8.18' + assert execution_request["input"]["arguments"][0]["key"] == "recipe" + assert ( + json.loads(execution_request["input"]["arguments"][0]["value"])["source"] + == json.loads( + '{"source":{"type":"mysql","config":{"include_tables":true,"database":null,"password":"${MYSQL_PASSWORD}","profiling":{"enabled":false},"host_port":null,"include_views":true,"username":"${MYSQL_USERNAME}"}},"pipeline_name":"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927"}' + )["source"] + ) + assert execution_request["input"]["arguments"][1]["key"] == "version" + assert execution_request["input"]["arguments"][1]["value"] == "0.8.18" # Verify no result assert execution_request["result"] is None # Now cleanup and remove the ingestion source - json = { + json_q = { "query": """mutation deleteIngestionSource($urn: String!) {\n deleteIngestionSource(urn: $urn) }""", - "variables": { - "urn": ingestion_source_urn - } + "variables": {"urn": ingestion_source_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json + f"{get_frontend_url()}/api/v2/graphql", json=json_q ) response.raise_for_status() res_data = response.json() @@ -572,4 +539,3 @@ def test_create_list_get_ingestion_execution_request(frontend_session): assert res_data["data"] assert res_data["data"]["deleteIngestionSource"] is not None assert "errors" not in res_data - diff --git a/smoke-test/tests/policies/__init__.py b/smoke-test/tests/policies/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/smoke-test/tests/policies/test_policies.py b/smoke-test/tests/policies/test_policies.py new file mode 100644 index 00000000000000..eefc09a7113143 --- /dev/null +++ b/smoke-test/tests/policies/test_policies.py @@ -0,0 +1,223 @@ +import time +import pytest +import requests +from tests.utils import get_frontend_url, wait_for_healthcheck_util, get_admin_credentials + +TEST_POLICY_NAME = "Updated Platform Policy" + + +@pytest.fixture(scope="session") +def wait_for_healthchecks(): + wait_for_healthcheck_util() + yield + + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + + +@pytest.fixture(scope="session") +def frontend_session(wait_for_healthchecks): + session = requests.Session() + + headers = { + "Content-Type": "application/json", + } + (admin_user, admin_pass) = get_admin_credentials() + data = '{"username":"' + admin_user + '", "password":"' + admin_pass + '"}' + response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data) + response.raise_for_status() + + yield session + +@pytest.mark.dependency(depends=["test_healthchecks"]) +@pytest.fixture(scope='class', autouse=True) +def test_frontend_list_policies(frontend_session): + """Fixture to execute setup before and tear down after all tests are run""" + res_data = listPolicies(frontend_session) + + assert res_data + assert res_data["data"] + assert res_data["data"]["listPolicies"] + assert res_data["data"]["listPolicies"]["start"] == 0 + assert res_data["data"]["listPolicies"]["count"] > 0 + assert len(res_data["data"]["listPolicies"]["policies"]) > 0 + + # Verify that policy to be created does not exist before the test. + # If it does, this test class's state is tainted + result = filter( + lambda x: x["name"] == TEST_POLICY_NAME, + res_data["data"]["listPolicies"]["policies"], + ) + assert len(list(result)) == 0 + + # Run remaining tests. + yield + + res_data = listPolicies(frontend_session) + + assert res_data + assert res_data["data"] + assert res_data["data"]["listPolicies"] + + # Verify that policy that was created is no longer in the list + result = filter( + lambda x: x["name"] == TEST_POLICY_NAME, + res_data["data"]["listPolicies"]["policies"], + ) + assert len(list(result)) == 0 + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_frontend_policy_operations(frontend_session): + + json = { + "query": """mutation createPolicy($input: PolicyUpdateInput!) {\n + createPolicy(input: $input) }""", + "variables": { + "input": { + "type": "METADATA", + "name": "Test Metadata Policy", + "description": "My Metadaata Policy", + "state": "ACTIVE", + "resources": {"type": "dataset", "allResources": True}, + "privileges": ["EDIT_ENTITY_TAGS"], + "actors": { + "users": ["urn:li:corpuser:datahub"], + "resourceOwners": False, + "allUsers": False, + "allGroups": False, + }, + } + }, + } + + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["createPolicy"] + + new_urn = res_data["data"]["createPolicy"] + + # Sleep for eventual consistency + time.sleep(3) + + update_json = { + "query": """mutation updatePolicy($urn: String!, $input: PolicyUpdateInput!) {\n + updatePolicy(urn: $urn, input: $input) }""", + "variables": { + "urn": new_urn, + "input": { + "type": "METADATA", + "state": "ACTIVE", + "name": "Test Metadata Policy", + "description": "Updated Metadaata Policy", + "privileges": ["EDIT_ENTITY_TAGS", "EDIT_ENTITY_GLOSSARY_TERMS"], + "actors": { + "resourceOwners": False, + "allUsers": True, + "allGroups": False, + }, + }, + }, + } + + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=update_json) + response.raise_for_status() + res_data = response.json() + + # Check updated was submitted successfully + assert res_data + assert res_data["data"] + assert res_data["data"]["updatePolicy"] + assert res_data["data"]["updatePolicy"] == new_urn + + # Sleep for eventual consistency + time.sleep(3) + + res_data = listPolicies(frontend_session) + + assert res_data + assert res_data["data"] + assert res_data["data"]["listPolicies"] + + # Verify that the updated policy appears in the list and has the appropriate changes + result = list(filter( + lambda x: x["urn"] == new_urn, res_data["data"]["listPolicies"]["policies"] + )) + print(result) + + assert len(result) == 1 + assert result[0]["description"] == "Updated Metadaata Policy" + assert result[0]["privileges"] == ["EDIT_ENTITY_TAGS", "EDIT_ENTITY_GLOSSARY_TERMS"] + assert result[0]["actors"]["allUsers"] == True + + # Now test that the policy can be deleted + json = { + "query": """mutation deletePolicy($urn: String!) {\n + deletePolicy(urn: $urn) }""", + "variables": {"urn": new_urn}, + } + + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + response.raise_for_status() + res_data = response.json() + + res_data = listPolicies(frontend_session) + + assert res_data + assert res_data["data"] + assert res_data["data"]["listPolicies"] + + # Verify that the URN is no longer in the list + result = filter( + lambda x: x["urn"] == new_urn, + res_data["data"]["listPolicies"]["policies"], + ) + assert len(list(result)) == 0 + +def listPolicies(session): + json = { + "query": """query listPolicies($input: ListPoliciesInput!) {\n + listPolicies(input: $input) {\n + start\n + count\n + total\n + policies {\n + urn\n + type\n + name\n + description\n + state\n + resources {\n + type\n + allResources\n + resources\n + }\n + privileges\n + actors {\n + users\n + groups\n + allUsers\n + allGroups\n + resourceOwners\n + }\n + editable\n + }\n + }\n + }""", + "variables": { + "input": { + "start": "0", + "count": "20", + } + }, + } + response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + response.raise_for_status() + + return response.json() diff --git a/smoke-test/tests/pytest.ini b/smoke-test/tests/pytest.ini new file mode 100644 index 00000000000000..61ce840fd713c6 --- /dev/null +++ b/smoke-test/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + read_only: marks tests as read only (deselect with '-m "not read_only"') diff --git a/smoke-test/tests/read_only/__init__.py b/smoke-test/tests/read_only/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/smoke-test/tests/read_only/test_analytics.py b/smoke-test/tests/read_only/test_analytics.py new file mode 100644 index 00000000000000..ce547246267f03 --- /dev/null +++ b/smoke-test/tests/read_only/test_analytics.py @@ -0,0 +1,55 @@ +import pytest + +from tests.utils import get_frontend_url + + +@pytest.mark.read_only +def test_highlights_is_accessible(frontend_session): + json = { + "query": """ + query getHighlights { + getHighlights { + value + title + body + } + } + """, + } + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + res_json = response.json() + assert res_json, f"Received JSON was {res_json}" + + +@pytest.mark.read_only +def test_analytics_chart_is_accessible(frontend_session): + json = { + "query": """ + query getAnalyticsCharts { + getAnalyticsCharts { + groupId + title + } + } + """, + } + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + res_json = response.json() + assert res_json, f"Received JSON was {res_json}" + + +@pytest.mark.read_only +def test_metadata_analytics_chart_is_accessible(frontend_session): + json = { + "query": """ + query getMetadataAnalyticsCharts($input: MetadataAnalyticsInput!) { + getMetadataAnalyticsCharts(input: $input) { + groupId + title + } + } + """, + } + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + res_json = response.json() + assert res_json, f"Received JSON was {res_json}" diff --git a/smoke-test/tests/read_only/test_ingestion_list.py b/smoke-test/tests/read_only/test_ingestion_list.py new file mode 100644 index 00000000000000..17868b1b6080dc --- /dev/null +++ b/smoke-test/tests/read_only/test_ingestion_list.py @@ -0,0 +1,45 @@ +import pytest + +from tests.test_result_msg import add_datahub_stats +from tests.utils import get_frontend_url + + +@pytest.mark.read_only +def test_policies_are_accessible(frontend_session): + json = { + "query": """ + query listIngestionSources($input: ListIngestionSourcesInput!) { + listIngestionSources(input: $input) { + total + ingestionSources { + urn + name + type + config { + version + } + } + } + } + """, + "variables": {"input": {"query": "*"}}, + } + + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + res_json = response.json() + assert res_json, f"Received JSON was {res_json}" + + res_data = res_json.get("data", {}).get("listIngestionSources", {}) + assert res_data, f"Received listIngestionSources were {res_data}" + add_datahub_stats("num-ingestion-sources", res_data["total"]) + + if res_data["total"] > 0: + for ingestion_source in res_data.get("ingestionSources"): + name = ingestion_source.get("name") + source_type = ingestion_source.get("type") + urn = ingestion_source.get("urn") + version = ingestion_source.get("config", {}).get("version") + add_datahub_stats( + f"ingestion-source-{urn}", + {"name": name, "version": version, "source_type": source_type}, + ) diff --git a/smoke-test/tests/read_only/test_policies.py b/smoke-test/tests/read_only/test_policies.py new file mode 100644 index 00000000000000..8e4d6a4be29a49 --- /dev/null +++ b/smoke-test/tests/read_only/test_policies.py @@ -0,0 +1,32 @@ +import pytest + +from tests.test_result_msg import add_datahub_stats +from tests.utils import get_frontend_url + + +@pytest.mark.read_only +def test_policies_are_accessible(frontend_session): + json = { + "query": """ + query listPolicies($input: ListPoliciesInput!) { + listPolicies(input: $input) { + total + policies { + urn + name + state + } + } + } + """, + "variables": {"input": {"query": "*"}}, + } + + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + res_json = response.json() + assert res_json, f"Received JSON was {res_json}" + + res_data = res_json.get("data", {}).get("listPolicies", {}) + assert res_data, f"Received listPolicies were {res_data}" + assert res_data["total"] > 0, f"Total was {res_data['total']}" + add_datahub_stats("num-policies", res_data["total"]) diff --git a/smoke-test/tests/read_only/test_search.py b/smoke-test/tests/read_only/test_search.py new file mode 100644 index 00000000000000..79e8a8fea8c2dd --- /dev/null +++ b/smoke-test/tests/read_only/test_search.py @@ -0,0 +1,88 @@ +import pytest +import requests + +from tests.test_result_msg import add_datahub_stats +from tests.utils import get_frontend_url, get_gms_url + +restli_default_headers = { + "X-RestLi-Protocol-Version": "2.0.0", +} + + +def _get_search_result(entity: str): + json = {"input": "*", "entity": entity, "start": 0, "count": 1} + response = requests.post( + f"{get_gms_url()}/entities?action=search", + headers=restli_default_headers, + json=json, + ) + res_data = response.json() + assert res_data, f"response data was {res_data}" + assert res_data["value"], f"response data was {res_data}" + return res_data["value"] + + +@pytest.mark.read_only +@pytest.mark.parametrize( + "entity_type,api_name", + [ + ("chart", "chart"), + ("dataset", "dataset"), + ("dashboard", "dashboard"), + ( + # Task + "dataJob", + "dataJob", + ), + ( + # Pipeline + "dataFlow", + "dataFlow", + ), + ("container", "container"), + ("tag", "tag"), + ("corpUser", "corpUser"), + ("mlFeature", "mlFeature"), + ("glossaryTerm", "glossaryTerm"), + ("domain", "domain"), + ("mlPrimaryKey", "mlPrimaryKey"), + ("corpGroup", "corpGroup"), + ("mlFeatureTable", "mlFeatureTable"), + ( + # Term group + "glossaryNode", + "glossaryNode", + ), + ("mlModel", "mlModel"), + ], +) +def test_search_works(frontend_session, entity_type, api_name): + search_result = _get_search_result(entity_type) + num_entities = search_result["numEntities"] + add_datahub_stats(f"num-{entity_type}", num_entities) + if num_entities == 0: + print(f"[WARN] No results for {entity_type}") + return + entities = search_result["entities"] + + first_urn = entities[0]["entity"] + + json = { + "query": """ + query """ + + api_name + + """($input: String!) { + """ + + api_name + + """(urn: $input) { + urn + } + } + """, + "variables": {"input": first_urn}, + } + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) + response.raise_for_status() + res_data = response.json() + assert res_data["data"], f"res_data was {res_data}" + assert res_data["data"][api_name]["urn"] == first_urn, f"res_data was {res_data}" diff --git a/smoke-test/tests/read_only/test_services_up.py b/smoke-test/tests/read_only/test_services_up.py new file mode 100644 index 00000000000000..e48df52bb98642 --- /dev/null +++ b/smoke-test/tests/read_only/test_services_up.py @@ -0,0 +1,25 @@ +import os + +import pytest +import requests + +from tests.utils import get_gms_url, wait_for_healthcheck_util + +# Kept separate so that it does not cause failures in PRs +DATAHUB_VERSION = os.getenv("TEST_DATAHUB_VERSION") + + +@pytest.mark.read_only +def test_services_up(): + wait_for_healthcheck_util() + + +@pytest.mark.read_only +def test_gms_config_accessible(): + gms_config = requests.get(f"{get_gms_url()}/config").json() + assert gms_config is not None + + if DATAHUB_VERSION is not None: + assert gms_config["versions"]["linkedin/datahub"]["version"] == DATAHUB_VERSION + else: + print("[WARN] TEST_DATAHUB_VERSION is not set") diff --git a/smoke-test/tests/tags-and-terms/tags_and_terms_test.py b/smoke-test/tests/tags-and-terms/tags_and_terms_test.py index f043d10b0692a0..b0ca29b544cfef 100644 --- a/smoke-test/tests/tags-and-terms/tags_and_terms_test.py +++ b/smoke-test/tests/tags-and-terms/tags_and_terms_test.py @@ -1,7 +1,6 @@ import pytest -from tests.utils import FRONTEND_ENDPOINT -from tests.utils import ingest_file_via_rest -from tests.utils import delete_urns_from_file +from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest, wait_for_healthcheck_util + @pytest.fixture(scope="module", autouse=True) def ingest_cleanup_data(request): @@ -11,12 +10,23 @@ def ingest_cleanup_data(request): print("removing test data") delete_urns_from_file("tests/tags-and-terms/data.json") -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_add_tag(frontend_session,wait_for_healthchecks): + +@pytest.fixture(scope="session") +def wait_for_healthchecks(): + wait_for_healthcheck_util() + yield + + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_add_tag(frontend_session): platform = "urn:li:dataPlatform:kafka" - dataset_name = ( - "test-tags-terms-sample-kafka" - ) + dataset_name = "test-tags-terms-sample-kafka" env = "PROD" dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})" @@ -34,14 +44,12 @@ def test_add_tag(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } # Fetch tags response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -49,7 +57,7 @@ def test_add_tag(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["globalTags"] == None + assert res_data["data"]["dataset"]["globalTags"] is None add_json = { "query": """mutation addTag($input: TagAssociationInput!) {\n @@ -57,14 +65,14 @@ def test_add_tag(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "tagUrn": "urn:li:tag:Legacy", - "resourceUrn": dataset_urn, + "tagUrn": "urn:li:tag:Legacy", + "resourceUrn": dataset_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json + f"{get_frontend_url()}/api/v2/graphql", json=add_json ) response.raise_for_status() res_data = response.json() @@ -75,7 +83,7 @@ def test_add_tag(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -83,7 +91,17 @@ def test_add_tag(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["globalTags"] == {'tags': [{'tag': {'description': 'Indicates the dataset is no longer supported', 'name': 'Legacy', 'urn': 'urn:li:tag:Legacy'}}]} + assert res_data["data"]["dataset"]["globalTags"] == { + "tags": [ + { + "tag": { + "description": "Indicates the dataset is no longer supported", + "name": "Legacy", + "urn": "urn:li:tag:Legacy", + } + } + ] + } remove_json = { "query": """mutation removeTag($input: TagAssociationInput!) {\n @@ -91,14 +109,14 @@ def test_add_tag(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "tagUrn": "urn:li:tag:Legacy", - "resourceUrn": dataset_urn, + "tagUrn": "urn:li:tag:Legacy", + "resourceUrn": dataset_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json + f"{get_frontend_url()}/api/v2/graphql", json=remove_json ) response.raise_for_status() res_data = response.json() @@ -111,7 +129,7 @@ def test_add_tag(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -119,10 +137,11 @@ def test_add_tag(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["globalTags"] == {'tags': [] } + assert res_data["data"]["dataset"]["globalTags"] == {"tags": []} -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_add_tag_to_chart(frontend_session): chart_urn = "urn:li:chart:(looker,test-tags-terms-sample-chart)" chart_json = { @@ -139,14 +158,12 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": chart_urn - } + "variables": {"urn": chart_urn}, } # Fetch tags response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=chart_json + f"{get_frontend_url()}/api/v2/graphql", json=chart_json ) response.raise_for_status() res_data = response.json() @@ -154,7 +171,7 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["chart"] - assert res_data["data"]["chart"]["globalTags"] == None + assert res_data["data"]["chart"]["globalTags"] is None add_json = { "query": """mutation addTag($input: TagAssociationInput!) {\n @@ -162,14 +179,14 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "tagUrn": "urn:li:tag:Legacy", - "resourceUrn": chart_urn, + "tagUrn": "urn:li:tag:Legacy", + "resourceUrn": chart_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json + f"{get_frontend_url()}/api/v2/graphql", json=add_json ) response.raise_for_status() res_data = response.json() @@ -180,7 +197,7 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=chart_json + f"{get_frontend_url()}/api/v2/graphql", json=chart_json ) response.raise_for_status() res_data = response.json() @@ -188,7 +205,17 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["chart"] - assert res_data["data"]["chart"]["globalTags"] == {'tags': [{'tag': {'description': 'Indicates the dataset is no longer supported', 'name': 'Legacy', 'urn': 'urn:li:tag:Legacy'}}]} + assert res_data["data"]["chart"]["globalTags"] == { + "tags": [ + { + "tag": { + "description": "Indicates the dataset is no longer supported", + "name": "Legacy", + "urn": "urn:li:tag:Legacy", + } + } + ] + } remove_json = { "query": """mutation removeTag($input: TagAssociationInput!) {\n @@ -196,14 +223,14 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "tagUrn": "urn:li:tag:Legacy", - "resourceUrn": chart_urn, + "tagUrn": "urn:li:tag:Legacy", + "resourceUrn": chart_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json + f"{get_frontend_url()}/api/v2/graphql", json=remove_json ) response.raise_for_status() res_data = response.json() @@ -214,7 +241,7 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=chart_json + f"{get_frontend_url()}/api/v2/graphql", json=chart_json ) response.raise_for_status() res_data = response.json() @@ -222,14 +249,13 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["chart"] - assert res_data["data"]["chart"]["globalTags"] == {'tags': [] } + assert res_data["data"]["chart"]["globalTags"] == {"tags": []} -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_add_term(frontend_session,wait_for_healthchecks): + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_add_term(frontend_session): platform = "urn:li:dataPlatform:kafka" - dataset_name = ( - "test-tags-terms-sample-kafka" - ) + dataset_name = "test-tags-terms-sample-kafka" env = "PROD" dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})" @@ -246,15 +272,12 @@ def test_add_term(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } - # Fetch the terms response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -262,7 +285,7 @@ def test_add_term(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["glossaryTerms"] == None + assert res_data["data"]["dataset"]["glossaryTerms"] is None add_json = { "query": """mutation addTerm($input: TermAssociationInput!) {\n @@ -270,14 +293,14 @@ def test_add_term(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "termUrn": "urn:li:glossaryTerm:SavingAccount", - "resourceUrn": dataset_urn, + "termUrn": "urn:li:glossaryTerm:SavingAccount", + "resourceUrn": dataset_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json + f"{get_frontend_url()}/api/v2/graphql", json=add_json ) response.raise_for_status() res_data = response.json() @@ -290,7 +313,7 @@ def test_add_term(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -298,7 +321,16 @@ def test_add_term(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["glossaryTerms"] == {'terms': [{'term': {'name': 'SavingAccount', 'urn': 'urn:li:glossaryTerm:SavingAccount'}}]} + assert res_data["data"]["dataset"]["glossaryTerms"] == { + "terms": [ + { + "term": { + "name": "SavingAccount", + "urn": "urn:li:glossaryTerm:SavingAccount", + } + } + ] + } remove_json = { "query": """mutation removeTerm($input: TermAssociationInput!) {\n @@ -306,14 +338,14 @@ def test_add_term(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "termUrn": "urn:li:glossaryTerm:SavingAccount", - "resourceUrn": dataset_urn, + "termUrn": "urn:li:glossaryTerm:SavingAccount", + "resourceUrn": dataset_urn, } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json + f"{get_frontend_url()}/api/v2/graphql", json=remove_json ) response.raise_for_status() res_data = response.json() @@ -325,7 +357,7 @@ def test_add_term(frontend_session,wait_for_healthchecks): assert res_data["data"]["removeTerm"] is True # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json + f"{get_frontend_url()}/api/v2/graphql", json=dataset_json ) response.raise_for_status() res_data = response.json() @@ -333,14 +365,13 @@ def test_add_term(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["glossaryTerms"] == {'terms': []} + assert res_data["data"]["dataset"]["glossaryTerms"] == {"terms": []} + -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_update_schemafield(frontend_session,wait_for_healthchecks): +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_update_schemafield(frontend_session): platform = "urn:li:dataPlatform:kafka" - dataset_name = ( - "test-tags-terms-sample-kafka" - ) + dataset_name = "test-tags-terms-sample-kafka" env = "PROD" dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})" @@ -367,12 +398,10 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } - dataset_schema_json_tags = { + dataset_schema_json_tags = { "query": """query getDataset($urn: String!) {\n dataset(urn: $urn) {\n urn\n @@ -396,12 +425,10 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } - dataset_schema_json_description = { + dataset_schema_json_description = { "query": """query getDataset($urn: String!) {\n dataset(urn: $urn) {\n urn\n @@ -417,14 +444,12 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "urn": dataset_urn - } + "variables": {"urn": dataset_urn}, } # dataset schema tags response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags ) response.raise_for_status() res_data = response.json() @@ -432,7 +457,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == None + assert res_data["data"]["dataset"]["editableSchemaMetadata"] is None add_json = { "query": """mutation addTag($input: TagAssociationInput!) {\n @@ -440,16 +465,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "tagUrn": "urn:li:tag:Legacy", - "resourceUrn": dataset_urn, - "subResource": "[version=2.0].[type=boolean].field_bar", - "subResourceType": "DATASET_FIELD" + "tagUrn": "urn:li:tag:Legacy", + "resourceUrn": dataset_urn, + "subResource": "[version=2.0].[type=boolean].field_bar", + "subResourceType": "DATASET_FIELD", } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json + f"{get_frontend_url()}/api/v2/graphql", json=add_json ) response.raise_for_status() res_data = response.json() @@ -460,7 +485,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): # Refetch the dataset schema response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags ) response.raise_for_status() res_data = response.json() @@ -468,7 +493,23 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'globalTags': {'tags': [{'tag': {'description': 'Indicates the dataset is no longer supported', 'name': 'Legacy', 'urn': 'urn:li:tag:Legacy'}}]}}]} + assert res_data["data"]["dataset"]["editableSchemaMetadata"] == { + "editableSchemaFieldInfo": [ + { + "globalTags": { + "tags": [ + { + "tag": { + "description": "Indicates the dataset is no longer supported", + "name": "Legacy", + "urn": "urn:li:tag:Legacy", + } + } + ] + } + } + ] + } remove_json = { "query": """mutation removeTag($input: TagAssociationInput!) {\n @@ -476,16 +517,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "tagUrn": "urn:li:tag:Legacy", - "resourceUrn": dataset_urn, - "subResource": "[version=2.0].[type=boolean].field_bar", - "subResourceType": "DATASET_FIELD" + "tagUrn": "urn:li:tag:Legacy", + "resourceUrn": dataset_urn, + "subResource": "[version=2.0].[type=boolean].field_bar", + "subResourceType": "DATASET_FIELD", } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json + f"{get_frontend_url()}/api/v2/graphql", json=remove_json ) response.raise_for_status() res_data = response.json() @@ -498,7 +539,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags ) response.raise_for_status() res_data = response.json() @@ -506,7 +547,9 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'globalTags': {'tags': []}}]} + assert res_data["data"]["dataset"]["editableSchemaMetadata"] == { + "editableSchemaFieldInfo": [{"globalTags": {"tags": []}}] + } add_json = { "query": """mutation addTerm($input: TermAssociationInput!) {\n @@ -514,16 +557,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "termUrn": "urn:li:glossaryTerm:SavingAccount", - "resourceUrn": dataset_urn, - "subResource": "[version=2.0].[type=boolean].field_bar", - "subResourceType": "DATASET_FIELD" + "termUrn": "urn:li:glossaryTerm:SavingAccount", + "resourceUrn": dataset_urn, + "subResource": "[version=2.0].[type=boolean].field_bar", + "subResourceType": "DATASET_FIELD", } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json + f"{get_frontend_url()}/api/v2/graphql", json=add_json ) response.raise_for_status() res_data = response.json() @@ -534,7 +577,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): # Refetch the dataset schema response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_terms + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_terms ) response.raise_for_status() res_data = response.json() @@ -542,7 +585,22 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'glossaryTerms': {'terms': [{'term': {'name': 'SavingAccount', 'urn': 'urn:li:glossaryTerm:SavingAccount'}}]}}]} + assert res_data["data"]["dataset"]["editableSchemaMetadata"] == { + "editableSchemaFieldInfo": [ + { + "glossaryTerms": { + "terms": [ + { + "term": { + "name": "SavingAccount", + "urn": "urn:li:glossaryTerm:SavingAccount", + } + } + ] + } + } + ] + } remove_json = { "query": """mutation removeTerm($input: TermAssociationInput!) {\n @@ -550,16 +608,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "termUrn": "urn:li:glossaryTerm:SavingAccount", - "resourceUrn": dataset_urn, - "subResource": "[version=2.0].[type=boolean].field_bar", - "subResourceType": "DATASET_FIELD" + "termUrn": "urn:li:glossaryTerm:SavingAccount", + "resourceUrn": dataset_urn, + "subResource": "[version=2.0].[type=boolean].field_bar", + "subResourceType": "DATASET_FIELD", } - } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json + f"{get_frontend_url()}/api/v2/graphql", json=remove_json ) response.raise_for_status() res_data = response.json() @@ -570,7 +628,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_terms + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_terms ) response.raise_for_status() res_data = response.json() @@ -578,11 +636,13 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'glossaryTerms': {'terms': []}}]} + assert res_data["data"]["dataset"]["editableSchemaMetadata"] == { + "editableSchemaFieldInfo": [{"glossaryTerms": {"terms": []}}] + } # dataset schema tags response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags ) response.raise_for_status() res_data = response.json() @@ -593,17 +653,17 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): }""", "variables": { "input": { - "description": "new description", - "resourceUrn": dataset_urn, - "subResource": "[version=2.0].[type=boolean].field_bar", - "subResourceType": "DATASET_FIELD" + "description": "new description", + "resourceUrn": dataset_urn, + "subResource": "[version=2.0].[type=boolean].field_bar", + "subResourceType": "DATASET_FIELD", } - } + }, } # fetch no description response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_description + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_description ) response.raise_for_status() res_data = response.json() @@ -611,10 +671,12 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{ 'description': None }]} + assert res_data["data"]["dataset"]["editableSchemaMetadata"] == { + "editableSchemaFieldInfo": [{"description": None}] + } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_description_json + f"{get_frontend_url()}/api/v2/graphql", json=update_description_json ) response.raise_for_status() res_data = response.json() @@ -625,7 +687,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): # Refetch the dataset response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_description + f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_description ) response.raise_for_status() res_data = response.json() @@ -633,4 +695,6 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["dataset"] - assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'description': 'new description'}]} + assert res_data["data"]["dataset"]["editableSchemaMetadata"] == { + "editableSchemaFieldInfo": [{"description": "new description"}] + } diff --git a/smoke-test/tests/test_result_msg.py b/smoke-test/tests/test_result_msg.py new file mode 100644 index 00000000000000..3ebd8e6dad6091 --- /dev/null +++ b/smoke-test/tests/test_result_msg.py @@ -0,0 +1,37 @@ +from slack_sdk import WebClient +import os +import asyncio +from datahub.upgrade.upgrade import retrieve_version_stats + + +datahub_stats = {} + + +def add_datahub_stats(stat_name, stat_val): + datahub_stats[stat_name] = stat_val + + +def send_to_slack(passed: str): + slack_api_token = os.getenv('SLACK_API_TOKEN') + slack_channel = os.getenv('SLACK_CHANNEL') + test_identifier = os.getenv('TEST_IDENTIFIER', 'LOCAL_TEST') + if slack_api_token is None or slack_channel is None: + return + client = WebClient(token=slack_api_token) + + key: str + message = "" + for key, val in datahub_stats.items(): + if key.startswith("num-"): + entity_type = key.replace("num-", "") + message += f"Num {entity_type} is {val}\n" + + client.chat_postMessage(channel=slack_channel, text=f'{test_identifier} Status - {passed}\n{message}') + + +def send_message(exitstatus): + try: + send_to_slack('PASSED' if exitstatus == 0 else 'FAILED') + except Exception as e: + # We don't want to fail pytest at all + print(f"Exception happened for sending msg to slack {e}") diff --git a/smoke-test/tests/test_stateful_ingestion.py b/smoke-test/tests/test_stateful_ingestion.py index 85c8477e0875d3..cfd311578160d1 100644 --- a/smoke-test/tests/test_stateful_ingestion.py +++ b/smoke-test/tests/test_stateful_ingestion.py @@ -3,12 +3,13 @@ from datahub.ingestion.api.committable import StatefulCommittable from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource -from datahub.ingestion.source.sql.sql_common import \ - BaseSQLAlchemyCheckpointState +from datahub.ingestion.source.sql.sql_common import BaseSQLAlchemyCheckpointState from datahub.ingestion.source.state.checkpoint import Checkpoint from sqlalchemy import create_engine from sqlalchemy.sql import text +from tests.utils import get_gms_url, get_mysql_url, get_mysql_username, get_mysql_password + def test_stateful_ingestion(wait_for_healthchecks): def create_mysql_engine(mysql_source_config_dict: Dict[str, Any]) -> Any: @@ -49,15 +50,16 @@ def get_current_checkpoint_from_pipeline( ) source_config_dict: Dict[str, Any] = { - "username": "datahub", - "password": "datahub", + "host_port": get_mysql_url(), + "username": get_mysql_username(), + "password": get_mysql_password(), "database": "datahub", "stateful_ingestion": { "enabled": True, "remove_stale_metadata": True, "state_provider": { "type": "datahub", - "config": {"datahub_api": {"server": "http://localhost:8080"}}, + "config": {"datahub_api": {"server": get_gms_url()}}, }, }, } @@ -69,13 +71,13 @@ def get_current_checkpoint_from_pipeline( }, "sink": { "type": "datahub-rest", - "config": {"server": "http://localhost:8080"}, + "config": {"server": get_gms_url()}, }, "pipeline_name": "mysql_stateful_ingestion_smoke_test_pipeline", "reporting": [ { "type": "datahub", - "config": {"datahub_api": {"server": "http://localhost:8080"}}, + "config": {"datahub_api": {"server": get_gms_url()}}, } ], } diff --git a/smoke-test/tests/tests/tests_test.py b/smoke-test/tests/tests/tests_test.py index b677d0086189ab..e0c460feb66ee0 100644 --- a/smoke-test/tests/tests/tests_test.py +++ b/smoke-test/tests/tests/tests_test.py @@ -1,8 +1,7 @@ import pytest -import time -from tests.utils import FRONTEND_ENDPOINT -from tests.utils import ingest_file_via_rest -from tests.utils import delete_urns_from_file + +from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest + @pytest.fixture(scope="module", autouse=True) def ingest_cleanup_data(request): @@ -12,17 +11,20 @@ def ingest_cleanup_data(request): print("removing test data") delete_urns_from_file("tests/tests/data.json") + @pytest.mark.dependency() def test_healthchecks(wait_for_healthchecks): # Call to wait_for_healthchecks fixture will do the actual functionality. pass + test_id = "test id" test_name = "test name" test_category = "test category" test_description = "test description" test_description = "test description" + def create_test(frontend_session): # Create new Test @@ -31,20 +33,18 @@ def create_test(frontend_session): createTest(input: $input) }""", "variables": { - "input": { - "id": test_id, - "name": test_name, - "category": test_category, - "description": test_description, - "definition": { - "json": "{}" - } - } - } + "input": { + "id": test_id, + "name": test_name, + "category": test_category, + "description": test_description, + "definition": {"json": "{}"}, + } + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=create_test_json + f"{get_frontend_url()}/api/v2/graphql", json=create_test_json ) response.raise_for_status() res_data = response.json() @@ -56,23 +56,23 @@ def create_test(frontend_session): return res_data["data"]["createTest"] + def delete_test(frontend_session, test_urn): delete_test_json = { "query": """mutation deleteTest($urn: String!) {\n deleteTest(urn: $urn) }""", - "variables": { - "urn": test_urn - } + "variables": {"urn": test_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=delete_test_json + f"{get_frontend_url()}/api/v2/graphql", json=delete_test_json ) response.raise_for_status() + @pytest.mark.dependency(depends=["test_healthchecks"]) -def test_create_test(frontend_session,wait_for_healthchecks): +def test_create_test(frontend_session, wait_for_healthchecks): test_urn = create_test(frontend_session) @@ -89,12 +89,10 @@ def test_create_test(frontend_session,wait_for_healthchecks): }\n } }""", - "variables": { - "urn": test_urn - } + "variables": {"urn": test_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_test_json + f"{get_frontend_url()}/api/v2/graphql", json=get_test_json ) response.raise_for_status() res_data = response.json() @@ -102,13 +100,13 @@ def test_create_test(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["test"] == { - "urn": test_urn, - "name": test_name, - "category": test_category, - "description": test_description, - "definition": { - "json": "{}", - } + "urn": test_urn, + "name": test_name, + "category": test_category, + "description": test_description, + "definition": { + "json": "{}", + }, } assert "errors" not in res_data @@ -117,7 +115,7 @@ def test_create_test(frontend_session,wait_for_healthchecks): # Ensure the test no longer exists response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_test_json + f"{get_frontend_url()}/api/v2/graphql", json=get_test_json ) response.raise_for_status() res_data = response.json() @@ -127,7 +125,7 @@ def test_create_test(frontend_session,wait_for_healthchecks): @pytest.mark.dependency(depends=["test_healthchecks", "test_create_test"]) -def test_update_test(frontend_session,wait_for_healthchecks): +def test_update_test(frontend_session, wait_for_healthchecks): test_urn = create_test(frontend_session) test_name = "new name" test_category = "new category" @@ -140,20 +138,18 @@ def test_update_test(frontend_session,wait_for_healthchecks): updateTest(urn: $urn, input: $input) }""", "variables": { - "urn": test_urn, - "input": { - "name": test_name, - "category": test_category, - "description": test_description, - "definition": { - "json": "{}" - } - } - } + "urn": test_urn, + "input": { + "name": test_name, + "category": test_category, + "description": test_description, + "definition": {"json": "{}"}, + }, + }, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_test_json + f"{get_frontend_url()}/api/v2/graphql", json=update_test_json ) response.raise_for_status() res_data = response.json() @@ -176,12 +172,10 @@ def test_update_test(frontend_session,wait_for_healthchecks): }\n } }""", - "variables": { - "urn": test_urn - } + "variables": {"urn": test_urn}, } response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_test_json + f"{get_frontend_url()}/api/v2/graphql", json=get_test_json ) response.raise_for_status() res_data = response.json() @@ -189,22 +183,23 @@ def test_update_test(frontend_session,wait_for_healthchecks): assert res_data assert res_data["data"] assert res_data["data"]["test"] == { - "urn": test_urn, - "name": test_name, - "category": test_category, - "description": test_description, - "definition": { - "json": "{}", - } + "urn": test_urn, + "name": test_name, + "category": test_category, + "description": test_description, + "definition": { + "json": "{}", + }, } assert "errors" not in res_data delete_test(frontend_session, test_urn) + @pytest.mark.dependency(depends=["test_healthchecks", "test_update_test"]) -def test_list_tests(frontend_session,wait_for_healthchecks): - list_tests_json = { - "query": """query listTests($input: ListTestsInput!) {\n +def test_list_tests(frontend_session, wait_for_healthchecks): + list_tests_json = { + "query": """query listTests($input: ListTestsInput!) {\n listTests(input: $input) {\n start\n count\n @@ -214,30 +209,27 @@ def test_list_tests(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": { - "input": { - "start": "0", - "count": "20" - } - } - } - - response = frontend_session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=list_tests_json - ) - response.raise_for_status() - res_data = response.json() - - assert res_data - assert res_data["data"] - assert res_data["data"]["listTests"]["total"] >= 2 - assert len(res_data["data"]["listTests"]["tests"]) >= 2 - assert "errors" not in res_data + "variables": {"input": {"start": "0", "count": "20"}}, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=list_tests_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["listTests"]["total"] >= 2 + assert len(res_data["data"]["listTests"]["tests"]) >= 2 + assert "errors" not in res_data @pytest.mark.dependency(depends=["test_healthchecks"]) -def test_get_test_results(frontend_session,wait_for_healthchecks): - urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tests-sample,PROD)" # Test urn +def test_get_test_results(frontend_session, wait_for_healthchecks): + urn = ( + "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tests-sample,PROD)" # Test urn + ) json = { "query": """query getDataset($urn: String!) {\n dataset(urn: $urn) {\n @@ -258,9 +250,9 @@ def test_get_test_results(frontend_session,wait_for_healthchecks): }\n }\n }""", - "variables": {"urn": urn }, + "variables": {"urn": urn}, } - response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) + response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() res_data = response.json() @@ -269,20 +261,6 @@ def test_get_test_results(frontend_session,wait_for_healthchecks): assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]["urn"] == urn assert res_data["data"]["dataset"]["testResults"] == { - "failing": [ - { - "test": { - "urn": "urn:li:test:test-1" - }, - "type": "FAILURE" - } - ], - "passing": [ - { - "test": { - "urn": "urn:li:test:test-2" - }, - "type": "SUCCESS" - } - ] + "failing": [{"test": {"urn": "urn:li:test:test-1"}, "type": "FAILURE"}], + "passing": [{"test": {"urn": "urn:li:test:test-2"}, "type": "SUCCESS"}], } diff --git a/smoke-test/tests/tokens/revokable_access_token_test.py b/smoke-test/tests/tokens/revokable_access_token_test.py index 778b3c3a2ed6f5..915b1123f77a90 100644 --- a/smoke-test/tests/tokens/revokable_access_token_test.py +++ b/smoke-test/tests/tokens/revokable_access_token_test.py @@ -1,15 +1,103 @@ +import os import pytest -import time import requests -from tests.utils import FRONTEND_ENDPOINT from time import sleep -from tests.utils import ingest_file_via_rest -from datahub.cli.ingest_cli import get_session_and_host +from tests.utils import get_frontend_url, wait_for_healthcheck_util, get_admin_credentials + + +# Disable telemetry +os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false" + +(admin_user, admin_pass) = get_admin_credentials() + + +@pytest.fixture(scope="session") +def wait_for_healthchecks(): + wait_for_healthcheck_util() + yield + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + +@pytest.mark.dependency(depends=["test_healthchecks"]) +@pytest.fixture(scope='class', autouse=True) +def custom_user_setup(): + """Fixture to execute setup before and tear down after all tests are run""" + admin_session = loginAs(admin_user, admin_pass) + + res_data = removeUser(admin_session, "urn:li:corpuser:user") + assert res_data + assert "error" not in res_data + + # Test getting the invite token + get_invite_token_json = { + "query": """query getNativeUserInviteToken {\n + getNativeUserInviteToken{\n + inviteToken\n + }\n + }""" + } + + get_invite_token_response = admin_session.post(f"{get_frontend_url()}/api/v2/graphql", json=get_invite_token_json) + get_invite_token_response.raise_for_status() + get_invite_token_res_data = get_invite_token_response.json() + + assert get_invite_token_res_data + assert get_invite_token_res_data["data"] + invite_token = get_invite_token_res_data["data"]["getNativeUserInviteToken"]["inviteToken"] + assert invite_token is not None + assert "error" not in invite_token + + # Pass the invite token when creating the user + sign_up_json = { + "fullName": "Test User", + "email": "user", + "password": "user", + "title": "Date Engineer", + "inviteToken": invite_token + } + + sign_up_response = admin_session.post(f"{get_frontend_url()}/signUp", json=sign_up_json) + sign_up_response.raise_for_status() + assert sign_up_response + assert "error" not in sign_up_response + # Sleep for eventual consistency + sleep(3) + + # signUp will override the session cookie to the new user to be signed up. + admin_session.cookies.clear() + admin_session = loginAs(admin_user, admin_pass) + + # Make user created user is there. + res_data = listUsers(admin_session) + assert res_data["data"] + assert res_data["data"]["listUsers"] + assert {'username': 'user'} in res_data["data"]["listUsers"]["users"] + + yield + + # Delete created user + res_data = removeUser(admin_session, "urn:li:corpuser:user") + assert res_data + assert res_data['data'] + assert res_data['data']['removeUser'] == True + # Sleep for eventual consistency + sleep(3) + + # Make user created user is not there. + res_data = listUsers(admin_session) + assert res_data["data"] + assert res_data["data"]["listUsers"] + assert {'username': 'user'} not in res_data["data"]["listUsers"]["users"] + +@pytest.mark.dependency(depends=["test_healthchecks"]) @pytest.fixture(autouse=True) -def test_setup(): +def access_token_setup(): """Fixture to execute asserts before and after a test is run""" - admin_session = loginAs("datahub", "datahub") + admin_session = loginAs(admin_user, admin_pass) res_data = listAccessTokens(admin_session) assert res_data @@ -17,22 +105,19 @@ def test_setup(): assert res_data["data"]["listAccessTokens"]["total"] == 0 assert not res_data["data"]["listAccessTokens"]["tokens"] - ingest_file_via_rest("tests/tokens/revokable_test_data.json") - - sleep(5) - yield - sleep(5) - # Clean up res_data = listAccessTokens(admin_session) for metadata in res_data["data"]["listAccessTokens"]["tokens"]: - revokeAccessToken(admin_session, metadata["id"]) + revokeAccessToken(admin_session, metadata["id"]) + + # Sleep for eventual consistency + sleep(3) -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_admin_can_create_list_and_revoke_tokens(): - admin_session = loginAs("datahub", "datahub") +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_admin_can_create_list_and_revoke_tokens(wait_for_healthchecks): + admin_session = loginAs(admin_user, admin_pass) # Using a super account, there should be no tokens res_data = listAccessTokens(admin_session) @@ -42,13 +127,15 @@ def test_admin_can_create_list_and_revoke_tokens(): assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 # Using a super account, generate a token for itself. - res_data = generateAccessToken_v2(admin_session, "urn:li:corpuser:datahub") + res_data = generateAccessToken_v2(admin_session, f"urn:li:corpuser:{admin_user}") assert res_data assert res_data["data"] assert res_data["data"]["createAccessToken"] assert res_data["data"]["createAccessToken"]["accessToken"] - assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == "urn:li:corpuser:datahub" + assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == f"urn:li:corpuser:{admin_user}" admin_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"] + # Sleep for eventual consistency + sleep(3) # Using a super account, list the previously created token. res_data = listAccessTokens(admin_session) @@ -56,8 +143,8 @@ def test_admin_can_create_list_and_revoke_tokens(): assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 1 - assert res_data["data"]["listAccessTokens"]["tokens"][1]["actorUrn"] == "urn:li:corpuser:datahub" - assert res_data["data"]["listAccessTokens"]["tokens"][1]["ownerUrn"] == "urn:li:corpuser:datahub" + assert res_data["data"]["listAccessTokens"]["tokens"][0]["actorUrn"] == f"urn:li:corpuser:{admin_user}" + assert res_data["data"]["listAccessTokens"]["tokens"][0]["ownerUrn"] == f"urn:li:corpuser:{admin_user}" # Check that the super account can revoke tokens that it created res_data = revokeAccessToken(admin_session, admin_tokenId) @@ -65,6 +152,8 @@ def test_admin_can_create_list_and_revoke_tokens(): assert res_data["data"] assert res_data["data"]["revokeAccessToken"] assert res_data["data"]["revokeAccessToken"] == True + # Sleep for eventual consistency + sleep(3) # Using a super account, there should be no tokens res_data = listAccessTokens(admin_session) @@ -73,9 +162,9 @@ def test_admin_can_create_list_and_revoke_tokens(): assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_admin_can_create_and_revoke_tokens_for_other_user(): - admin_session = loginAs("datahub", "datahub") +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_admin_can_create_and_revoke_tokens_for_other_user(wait_for_healthchecks): + admin_session = loginAs(admin_user, admin_pass) # Using a super account, there should be no tokens res_data = listAccessTokens(admin_session) @@ -92,6 +181,8 @@ def test_admin_can_create_and_revoke_tokens_for_other_user(): assert res_data["data"]["createAccessToken"]["accessToken"] assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == "urn:li:corpuser:user" user_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"] + # Sleep for eventual consistency + sleep(3) # Using a super account, list the previously created tokens. res_data = listAccessTokens(admin_session) @@ -100,7 +191,7 @@ def test_admin_can_create_and_revoke_tokens_for_other_user(): assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 1 assert res_data["data"]["listAccessTokens"]["tokens"][0]["actorUrn"] == "urn:li:corpuser:user" - assert res_data["data"]["listAccessTokens"]["tokens"][0]["ownerUrn"] == "urn:li:corpuser:datahub" + assert res_data["data"]["listAccessTokens"]["tokens"][0]["ownerUrn"] == f"urn:li:corpuser:{admin_user}" # Check that the super account can revoke tokens that it created for another user res_data = revokeAccessToken(admin_session, user_tokenId) @@ -108,6 +199,8 @@ def test_admin_can_create_and_revoke_tokens_for_other_user(): assert res_data["data"] assert res_data["data"]["revokeAccessToken"] assert res_data["data"]["revokeAccessToken"] == True + # Sleep for eventual consistency + sleep(3) # Using a super account, there should be no tokens res_data = listAccessTokens(admin_session) @@ -115,11 +208,10 @@ def test_admin_can_create_and_revoke_tokens_for_other_user(): assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 -""" -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_non_admin_can_create_list_revoke_tokens(): + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_non_admin_can_create_list_revoke_tokens(wait_for_healthchecks): user_session = loginAs("user", "user") - admin_session = loginAs("datahub", "datahub") # Normal user should be able to generate token for himself. res_data = generateAccessToken_v2(user_session, "urn:li:corpuser:user") @@ -129,9 +221,11 @@ def test_non_admin_can_create_list_revoke_tokens(): assert res_data["data"]["createAccessToken"]["accessToken"] assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == "urn:li:corpuser:user" user_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"] + # Sleep for eventual consistency + sleep(3) # User should be able to list his own token - res_data = listAccessTokens(user_session, [{"field": "actorUrn","value": "urn:li:corpuser:user"}]) + res_data = listAccessTokens(user_session, [{"field": "ownerUrn","value": "urn:li:corpuser:user"}]) assert res_data assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None @@ -146,18 +240,19 @@ def test_non_admin_can_create_list_revoke_tokens(): assert res_data["data"] assert res_data["data"]["revokeAccessToken"] assert res_data["data"]["revokeAccessToken"] == True + # Sleep for eventual consistency + sleep(3) # Using a normal account, check that all its tokens where removed. - res_data = listAccessTokens(user_session, [{"field": "actorUrn","value": "urn:li:corpuser:user"}]) + res_data = listAccessTokens(user_session, [{"field": "ownerUrn","value": "urn:li:corpuser:user"}]) assert res_data assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_admin_can_manage_tokens_generated_by_other_user(): - user_session = loginAs("user", "user") - admin_session = loginAs("datahub", "datahub") +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_admin_can_manage_tokens_generated_by_other_user(wait_for_healthchecks): + admin_session = loginAs(admin_user, admin_pass) # Using a super account, there should be no tokens res_data = listAccessTokens(admin_session) @@ -166,17 +261,23 @@ def test_admin_can_manage_tokens_generated_by_other_user(): assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 - # Normal user should be able to generate token for himself. + admin_session.cookies.clear() + user_session = loginAs("user", "user") res_data = generateAccessToken_v2(user_session, "urn:li:corpuser:user") assert res_data assert res_data["data"] assert res_data["data"]["createAccessToken"] assert res_data["data"]["createAccessToken"]["accessToken"] assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == "urn:li:corpuser:user" + assert res_data["data"]["createAccessToken"]["metadata"]["ownerUrn"] == "urn:li:corpuser:user" user_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"] + # Sleep for eventual consistency + sleep(3) # Admin should be able to list other tokens - res_data = listAccessTokens(admin_session, [{"field": "actorUrn","value": "urn:li:corpuser:user"}]) + user_session.cookies.clear() + admin_session = loginAs(admin_user, admin_pass) + res_data = listAccessTokens(admin_session, [{"field": "ownerUrn","value": "urn:li:corpuser:user"}]) assert res_data assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None @@ -186,60 +287,44 @@ def test_admin_can_manage_tokens_generated_by_other_user(): assert res_data["data"]["listAccessTokens"]["tokens"][0]["id"] == user_tokenId # Admin can delete token created by someone else. + admin_session.cookies.clear() + admin_session = loginAs(admin_user, admin_pass) res_data = revokeAccessToken(admin_session, user_tokenId) assert res_data assert res_data["data"] assert res_data["data"]["revokeAccessToken"] assert res_data["data"]["revokeAccessToken"] == True + # Sleep for eventual consistency + sleep(3) # Using a normal account, check that all its tokens where removed. - res_data = listAccessTokens(user_session, [{"field": "actorUrn","value": "urn:li:corpuser:user"}]) + user_session.cookies.clear() + user_session = loginAs("user", "user") + res_data = listAccessTokens(user_session, [{"field": "ownerUrn","value": "urn:li:corpuser:user"}]) assert res_data assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 # Using the super account, check that all tokens where removed. - res_data = listAccessTokens(admin_session, [{"field": "actorUrn","value": "urn:li:corpuser:user"}]) + admin_session = loginAs(admin_user, admin_pass) + res_data = listAccessTokens(admin_session, [{"field": "ownerUrn","value": "urn:li:corpuser:user"}]) assert res_data assert res_data["data"] assert res_data["data"]["listAccessTokens"]["total"] is not None assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 -@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) -def test_non_admin_can_not_generate_tokens_for_others(): +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_non_admin_can_not_generate_tokens_for_others(wait_for_healthchecks): user_session = loginAs("user", "user") - # Normal user should not be able to generate token for another user - res_data = generateAccessToken_v2(user_session, "urn:li:corpuser:datahub") + # Normal user should not be able to generate token for another user + res_data = generateAccessToken_v2(user_session, f"urn:li:corpuser:{admin_user}") assert res_data assert res_data["errors"] assert res_data["errors"][0]["message"] == "Unauthorized to perform this action. Please contact your DataHub administrator." -""" -def generateAccessToken_v1(session, actorUrn): - # Create new token - json = { - "query": """query getAccessToken($input: GetAccessTokenInput!) {\n - getAccessToken(input: $input) {\n - accessToken\n - }\n - }""", - "variables": { - "input": { - "type": "PERSONAL", - "actorUrn": actorUrn, - "duration": "ONE_HOUR" - } - } - } - - response = session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) - response.raise_for_status() - return response.json() def generateAccessToken_v2(session, actorUrn): - # Create new token + # Create new token json = { "query": """mutation createAccessToken($input: CreateAccessTokenInput!) {\n createAccessToken(input: $input) {\n @@ -254,32 +339,29 @@ def generateAccessToken_v2(session, actorUrn): }\n }""", "variables": { - "input": { - "type": "PERSONAL", - "actorUrn": actorUrn, - "duration": "ONE_HOUR", - "name": "my token" - } - } + "input": { + "type": "PERSONAL", + "actorUrn": actorUrn, + "duration": "ONE_HOUR", + "name": "my token", + } + }, } - response = session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) + response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() - - sleep(5) return response.json() + def listAccessTokens(session, filters=[]): # Get count of existing tokens input = { - "start": "0", - "count": "20", + "start": "0", + "count": "20", } if filters: - input['filters'] = filters + input["filters"] = filters json = { "query": """query listAccessTokens($input: ListAccessTokenInput!) {\n @@ -295,45 +377,87 @@ def listAccessTokens(session, filters=[]): }\n }\n }""", - "variables": { - "input": input - } + "variables": {"input": input}, } - response = session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) + response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() return response.json() + def revokeAccessToken(session, tokenId): # Revoke token json = { "query": """mutation revokeAccessToken($tokenId: String!) {\n revokeAccessToken(tokenId: $tokenId) }""", - "variables": { - "tokenId": tokenId - } + "variables": {"tokenId": tokenId}, } - response = session.post( - f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json - ) - sleep(5) + response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json) response.raise_for_status() + return response.json() + def loginAs(username, password): session = requests.Session() headers = { "Content-Type": "application/json", } - data = '{"username":"' + username +'", "password":"' + password + '"}' + data = '{"username":"' + username + '", "password":"' + password + '"}' + response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data) + response.raise_for_status() + + return session + + +def removeUser(session, urn): + # Remove user + json = { + "query": """mutation removeUser($urn: String!) {\n + removeUser(urn: $urn) + }""", + "variables": { + "urn": urn + } + } + response = session.post( - f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data + f"{get_frontend_url()}/api/v2/graphql", json=json ) + response.raise_for_status() + return response.json() + + +def listUsers(session): + input = { + "start": "0", + "count": "20", + } + + # list users + json = { + "query": """query listUsers($input: ListUsersInput!) {\n + listUsers(input: $input) {\n + start\n + count\n + total\n + users {\n + username\n + } + } + }""", + "variables": { + "input": input + } + } - return session \ No newline at end of file + response = session.post( + f"{get_frontend_url()}/api/v2/graphql", json=json + ) + + response.raise_for_status() + return response.json() diff --git a/smoke-test/tests/utils.py b/smoke-test/tests/utils.py index 354a98c8e473d9..b3a33b3dd85cff 100644 --- a/smoke-test/tests/utils.py +++ b/smoke-test/tests/utils.py @@ -1,12 +1,74 @@ import json +import os +from typing import Any, Tuple import requests -from typing import Any from datahub.cli import cli_utils from datahub.ingestion.run.pipeline import Pipeline +from datahub.cli.docker import check_local_docker_containers + +def get_admin_credentials(): + return ("datahub", "datahub") + +def get_gms_url(): + return os.getenv("DATAHUB_GMS_URL") or "http://localhost:8080" + + +def get_frontend_url(): + return os.getenv("DATAHUB_FRONTEND_URL") or "http://localhost:9002" + + +def get_kafka_broker_url(): + return os.getenv("DATAHUB_KAFKA_URL") or "localhost:9092" + + +def get_kafka_schema_registry(): + return os.getenv("DATAHUB_KAFKA_SCHEMA_REGISTRY_URL") or "http://localhost:8081" + + +def get_mysql_url(): + return os.getenv("DATAHUB_MYSQL_URL") or "localhost:3306" + + +def get_mysql_username(): + return os.getenv("DATAHUB_MYSQL_USERNAME") or "datahub" + + +def get_mysql_password(): + return os.getenv("DATAHUB_MYSQL_PASSWORD") or "datahub" + + +def get_sleep_info() -> Tuple[int, int]: + return ( + int(os.getenv("DATAHUB_TEST_SLEEP_BETWEEN", 20)), + int(os.getenv("DATAHUB_TEST_SLEEP_TIMES", 15)), + ) + + +def is_k8s_enabled(): + return os.getenv("K8S_CLUSTER_ENABLED", "false").lower() in ["true", "yes"] + + +def wait_for_healthcheck_util(): + if is_k8s_enabled(): + # Simply assert that kubernetes endpoints are healthy, but don't wait. + assert not check_endpoint(f"{get_frontend_url()}/admin") + assert not check_endpoint(f"{get_gms_url()}/health") + else: + # Simply assert that docker is healthy, but don't wait. + assert not check_local_docker_containers() + + +def check_endpoint(url): + try: + get = requests.get(url) + if get.status_code == 200: + return + else: + return f"{url}: is Not reachable, status_code: {get.status_code}" + except requests.exceptions.RequestException as e: + raise SystemExit(f"{url}: is Not reachable \nErr: {e}") -GMS_ENDPOINT = "http://localhost:8080" -FRONTEND_ENDPOINT = "http://localhost:9002" def ingest_file_via_rest(filename: str) -> Any: pipeline = Pipeline.create( @@ -17,7 +79,7 @@ def ingest_file_via_rest(filename: str) -> Any: }, "sink": { "type": "datahub-rest", - "config": {"server": GMS_ENDPOINT}, + "config": {"server": get_gms_url()}, }, } ) @@ -39,19 +101,19 @@ def delete_urns_from_file(filename: str) -> None: with open(filename) as f: d = json.load(f) for entry in d: - is_mcp = 'entityUrn' in entry + is_mcp = "entityUrn" in entry urn = None # Kill Snapshot if is_mcp: - urn = entry['entityUrn'] + urn = entry["entityUrn"] else: - snapshot_union = entry['proposedSnapshot'] - snapshot = list(snapshot_union.values())[0] - urn = snapshot['urn'] + snapshot_union = entry["proposedSnapshot"] + snapshot = list(snapshot_union.values())[0] + urn = snapshot["urn"] payload_obj = {"urn": urn} cli_utils.post_delete_endpoint_with_session_and_url( session, - GMS_ENDPOINT + "/entities?action=delete", + get_gms_url() + "/entities?action=delete", payload_obj, ) diff --git a/temp_time_series/create_mock_index.py b/temp_time_series/create_mock_index.py deleted file mode 100755 index 65997f7011f355..00000000000000 --- a/temp_time_series/create_mock_index.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -import os -from datetime import datetime -from typing import Optional, Generator, Tuple - -# import hashlib - -HOUR_IN_MS = 3600000 -DAY_IN_MS = 86400000 -START_DAY_IN_MS = int(datetime.now().timestamp() * 1000) - 5 * DAY_IN_MS - -CounterType = Optional[int] -NameType = Optional[str] -IndexRowType = Tuple[ - NameType, - CounterType, - CounterType, - NameType, - CounterType, - CounterType, - CounterType, - CounterType, - CounterType, - CounterType, -] - - -def day(n: int) -> int: - return START_DAY_IN_MS + n * DAY_IN_MS - - -class MockIndexGenerator: - INDEX_NAME = "mock_dataset_stats_aspect_v1" - - INDEX_FIELD_NAMES = [ - "urn", - "rowCount", - "columnCount", - "columnStats.key", - "columnStats.numNull", - "eventTimestampMillis", - "eventGranularity", - "partitionSpec.parition", - "partitionSpec.timeWindow.startTimeMillis", - "partitionSpec.timeWindow.granulatiry", - ] - - INDEX_FIELD_TYPES = [ - "keyword", - "long", - "long", - "keyword", - "long", - "date", - "long", - "keyword", - "date", - "long", - ] - - def __init__(self, start_days_in_ms, num_recs, num_cols): - self._start_days_in_ms = start_days_in_ms - self._num_recs = num_recs - self._num_cols = num_cols - self._stat_num_rows_start = 10000 - self._stat_num_cols_start = 50 - self._stat_num_nulls = 100 - - def _get_num_rows(self, i: int): - return self._stat_num_rows_start + (100 * i) - - def _get_num_cols(self, i: int): - return self._stat_num_cols_start + i - - def _get_num_nulls(self, i: int, c: int): - return self._stat_num_nulls + c + (10 * i) - - def _get_event_time_ms(self, i: int): - return self._start_days_in_ms + (i * HOUR_IN_MS) - - @staticmethod - def _get_index_row_json(row: IndexRowType) -> str: - return ",".join( - [ - f'"{field}" : "{value}"' - for field, value in zip(MockIndexGenerator.INDEX_FIELD_NAMES, row) - if value is not None - ] - ) - - def get_records(self) -> Generator[IndexRowType, None, None]: - for i in range(self._num_recs): - # emit one table record - yield self._get_index_row_json(( - "table_1", - self._get_num_rows(i), - self._get_num_cols(i), - None, - None, - self._get_event_time_ms(i), - HOUR_IN_MS, - None, - None, - None) - ) - # emit one record per column - for c in range(self._num_cols): - yield self._get_index_row_json(( - f"table_1", - None, - None, - f"col_{c}", - self._get_num_nulls(i, c), - self._get_event_time_ms(i), - HOUR_IN_MS, - None, - None, - None) - ) - - @staticmethod - def get_props_json() -> str: - return ",".join( - [ - f'"{field}" : {{ "type" : "{type}" }}' - for field, type in zip( - MockIndexGenerator.INDEX_FIELD_NAMES, - MockIndexGenerator.INDEX_FIELD_TYPES, - ) - ] - ) - - -def gen_index_schema() -> None: - properties_json = MockIndexGenerator.get_props_json() - index_schema_gen_cmd = ( - f"curl -v -XPUT http://localhost:9200/{MockIndexGenerator.INDEX_NAME} -H 'Content-Type: application/json' -d '" - + """ - { - "settings":{}, - "mappings":{ - "properties":{ """ - + f"{properties_json}" - + """ - } - } - }'""" - ) - print(index_schema_gen_cmd) - os.system(index_schema_gen_cmd) - - -def populate_index_data() -> None: - for id, row in enumerate( - MockIndexGenerator(START_DAY_IN_MS, 100, 20).get_records() - ): - # id = hashlib.md5(row.encode("utf-8")).hexdigest() - index_row_gen_command = ( - f"curl -v -XPUT http://localhost:9200/{MockIndexGenerator.INDEX_NAME}/_doc/{id} " - + "-H 'Content-Type: application/json' -d '{ " - + f"{row}" - + " }'" - ) - print(index_row_gen_command) - os.system(index_row_gen_command) - - -def generate() -> None: - #gen_index_schema() - populate_index_data() - - -if __name__ == "__main__": - generate()