config.yml

version: 2.1

filters: &filters
  filters:
    tags:
      only: /^v[0-9]+(\.[0-9]+)*$/

filters__tags: &filters__tags
  filters:
    branches:
      ignore: /.*/
    tags:
      only: /^v[0-9]+(\.[0-9]+)*$/

workflows:
  test:
    jobs:
      - cache:
          <<: *filters
      - build:
          <<: *filters
          requires:
            - cache
      - test--15:
          <<: *filters
          requires:
            - cache
      - test--14:
          <<: *filters
          requires:
            - cache
      - test--13:
          <<: *filters
          requires:
            - cache
      - test--12:
          <<: *filters
          requires:
            - cache
      - test--11:
          <<: *filters
          requires:
            - cache
      - test--10:
          <<: *filters
          requires:
            - cache
      - test--migrations:
          <<: *filters
          requires:
            - cache
      - test--tap-github:
          <<: *filters
          requires:
            - test--12
      - test-release:
          <<: *filters__tags
          requires:
            - test--tap-github
            - test--migrations
            - build
      - approve-release:
          <<: *filters__tags
          type: approval
          requires:
            - test-release
      - release:
          <<: *filters__tags
          requires:
            - approve-release

cache: &cache deps-v7-{{ checksum "poetry.lock" }}-{{ checksum ".circleci/config.yml" }}

py: &py python:3.7.15-bullseye

restore__cache: &restore__cache
  restore_cache:
    keys:
      - *cache

# Simple checkout command to pull external forks.
# The CircleCI util does not work without setting up SSH keys
# which we technically do not need for open-source repos.
checkout_command: &checkout_command
  run:
    name: checkout
    command: |
      git clone https://github.com/datamill-co/target-postgres .
      if [[ "$CIRCLE_BRANCH" =~ ^pull\/* ]]; then
        git fetch origin refs/pull/${CIRCLE_PR_NUMBER}/head
        git checkout ${CIRCLE_SHA1}
      else
        git checkout ${CIRCLE_BRANCH}
      fi

install_poetry: &install_poetry
  run:
    name: Install poetry
    command: |
      export POETRY_HOME=/opt/poetry
      python -m venv $POETRY_HOME
      $POETRY_HOME/bin/pip install -U pip
      $POETRY_HOME/bin/pip install poetry==1.2.2
      $POETRY_HOME/bin/poetry --version

test__base: &test__base
  working_directory: /code/
  steps:
    - *checkout_command
    - *restore__cache
    - *install_poetry
    - attach_workspace:
        at: "./"

    - run:
        name: Run Tests
        command: |
          source venv/target-postgres/bin/activate
          pytest --verbose tests/unit
        environment:
          POSTGRES_HOST: localhost
          POSTGRES_DATABASE: target_postgres_test
          POSTGRES_USERNAME: postgres
          POSTGRES_PASSWORD: postgres

    - store_artifacts:
        path: target/test-results
        destination: raw-test-output

jobs:
  cache:
    working_directory: /code/
    docker:
      - image: *py
    steps:
      - *checkout_command
      - *restore__cache
      - *install_poetry

      - run:
          name: Install target-postgres
          command: |
            python -m venv venv/target-postgres
            source venv/target-postgres/bin/activate
            pip install -U pip
            /opt/poetry/bin/poetry install --with tests
            deactivate

      - run:
          name: Install older versions of target-postgres for migration testing
          command: ./tests/migrations/scripts/install_schema_versions.sh

      - run:
          name: Install tap-github
          command: |
            python -m venv venv/tap-github
            source venv/tap-github/bin/activate
            pip install -U pip
            pip install git+https://github.com/MeltanoLabs/tap-github.git@v1.1.0
            deactivate

      - run:
          name: Install tap-postgres
          command: |
            python -m venv venv/tap-postgres
            source venv/tap-postgres/bin/activate
            pip install -U pip
            pip install tap-postgres
            deactivate

      - save_cache:
          key: *cache
          paths:
            - "./venv"
            - "/usr/local/bin"
            - "/usr/local/lib/python3.7/site-packages"
            - "/opt/poetry"

      - persist_to_workspace:
          root: "./"
          paths:
            - "./venv"

  test--15:
    <<: *test__base
    docker:
      - image: *py
      - image: postgres:15.0
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres

  test--14:
    <<: *test__base
    docker:
      - image: *py
      - image: postgres:14.5
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres

  test--13:
    <<: *test__base
    docker:
      - image: *py
      - image: postgres:13.8
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres

  test--12:
    <<: *test__base
    docker:
      - image: *py
      - image: postgres:12.12
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres

  test--11:
    <<: *test__base
    docker:
      - image: *py
      - image: postgres:11.17-bullseye
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres

  test--10:
    <<: *test__base
    docker:
      - image: *py
      - image: postgres:10.22-bullseye
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres

  test--tap-github:
    working_directory: /code/
    docker:
      - image: *py
      - image: postgres:12.12
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres
    steps:
      - *checkout_command
      - *restore__cache
      - attach_workspace:
          at: "./"

      - run:
          name: Setup artifacts folder
          command: mkdir -p /code/artifacts/data

      - run:
          name: Setup tap-github
          working_directory: /code/.circleci/integration/tap-github
          command: sed "s/REPLACE_ME/$TAP_GITHUB_TOKEN/" config-template.json > config.json

      - run:
          name: Tap -> Data
          command: |
            source venv/tap-github/bin/activate
            cd /code/.circleci/integration/tap-github

            tap-github --config config.json --catalog catalog.json > /code/artifacts/data/tap

            deactivate

      - run:
          name: Data -> Target
          command: |
            source venv/target-postgres/bin/activate
            pip install -U pip
            /opt/poetry/bin/poetry install
            cd /code/.circleci/integration

            cat /code/artifacts/data/tap | target-postgres --config target-config.json

            deactivate

      - run:
          name: Target -> Data
          command: |
            source venv/tap-postgres/bin/activate
            cd /code/.circleci/integration/tap-postgres

            tap-postgres --config config.json --discover > tmp-properties.json

            ## Select _every_ table found in properties.
            ##  row-count seems to only show up inside of the necessary metadata object...easier than multi-line-sed
            sed 's/"row-count": 0,/"row-count": 0,"selected":true,/g' tmp-properties.json > /code/artifacts/data/properties.json

            tap-postgres --config config.json --properties /code/artifacts/data/properties.json > /code/artifacts/data/target

            deactivate

      - run:
          name: Repeatability of Data -> Target
          command: |
            source venv/target-postgres/bin/activate
            pip install -U pip
            pip install .
            cd /code/.circleci/integration

            cat /code/artifacts/data/tap | target-postgres --config target-config.json

            deactivate

            cd /code/

            source venv/tap-postgres/bin/activate
            cd /code/.circleci/integration/tap-postgres

            tap-postgres --config config.json --discover > tmp-properties.json

            ## Select _every_ table found in properties.
            ##  row-count seems to only show up inside of the necessary metadata object...easier than multi-line-sed
            sed 's/"row-count": 0,/"row-count": 0,"selected":true,/g' tmp-properties.json > /code/artifacts/data/properties.json

            tap-postgres --config config.json --properties /code/artifacts/data/properties.json > /code/artifacts/data/target.repeated

            deactivate

            ## TODO: compare repeated data to insure that we only changed _sdc values
            # diff /code/artifacts/data/target /code/artifacts/data/target.repeated

      - store_artifacts:
          path: /code/artifacts

  test--migrations:
    working_directory: /code/
    docker:
      - image: *py
      - image: postgres:12.12
        environment:
          POSTGRES_DB: target_postgres_test
          POSTGRES_PASSWORD: postgres
    steps:
      - *checkout_command
      - *restore__cache
      - *install_poetry
      - attach_workspace:
          at: "./"

      - run:
          name: Run Tests
          command: |
            source venv/target-postgres/bin/activate
            pytest --verbose tests/migrations
          environment:
            POSTGRES_HOST: localhost
            POSTGRES_DATABASE: target_postgres_test
            POSTGRES_USERNAME: postgres
            POSTGRES_PASSWORD: postgres

      - store_artifacts:
          path: /code/tests/migrations/artifacts
          destination: raw-test-output

  build:
    working_directory: /code/
    docker:
      - image: *py
    steps:
      - *checkout_command
      - *restore__cache
      - attach_workspace:
          at: "./"
      - run:
          name: Build distribution
          command: |
            source venv/target-postgres/bin/activate

            pip install -U pip
            pip install --upgrade setuptools wheel twine

            /opt/poetry/bin/poetry build

            deactivate

      - persist_to_workspace:
          root: "./"
          paths:
            - "./dist"

  test-release:
    working_directory: /code/
    docker:
      - image: *py
    steps:
      - *checkout_command
      - *restore__cache
      - attach_workspace:
          at: "./"
      - run:
          name: Validate tag
          command: |
            export TAG=`echo $CIRCLE_TAG | sed 's/v//'`
            VERSION=`grep version pyproject.toml | sed 's/^.*version = "\(.*\)",.*$/\1/'`

            echo tag: $TAG equals version: $VERSION '?'

            [[ $TAG == $VERSION ]]
      - run:
          name: Install upload tools
          command: pip install --upgrade twine
      - run:
          name: Test Publish
          environment:
            TWINE_USERNAME: datamill
            TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
          command: |
            export TWINE_PASSWORD=$PYPI__PASSWORD__TEST
            twine upload ./dist/*

  release:
    working_directory: /code/
    docker:
      - image: *py
    steps:
      - *checkout_command
      - *restore__cache
      - attach_workspace:
          at: "./"
      - run:
          name: Install upload tools
          command: pip install --upgrade twine
      - run:
          name: Publish
          environment:
            TWINE_USERNAME: datamill
          command: |
            export TWINE_PASSWORD=$PYPI__PASSWORD
            twine upload ./dist/*