.gitlab-ci.yml

image: humancellatlas/dss-build-box
# The Docker image `humancellatlas/dss-build-box` is created through a manual process from
# `${DSS_HOME}/allspark.Dockerfile`. See the contents of `${DSS_HOME}/allspark.Dockerfile` # creation and usage instructions.

variables:
  GIT_SUBMODULE_STRATEGY: normal
  DSS_ES_TIMEOUT: 30
  DSS_UNITTEST_OPTS: "-v -b"
  GITHUB_API: "https://api.github.com"

stages:
  - trufflehog
  - test
  - deploy
  - fusillade
  - integration_test
  - scale_and_performance
  - release

before_script:
  - date && date -u
# TODO: figure out how to get the gitlab-runner to not clone the repo as root - Brian H
  - virtualenv ~/venv
  - source ~/venv/bin/activate
  - pip install -r requirements-dev.txt
  - source environment
  - if [[ -f "environment.$CI_COMMIT_REF_NAME" ]]; then
  -   source environment.$CI_COMMIT_REF_NAME
  - fi
  - scripts/dss-ops.py secrets get application_secrets.json > application_secrets.json
  - scripts/dss-ops.py secrets get gcp-credentials.json > gcp-credentials.json
  - export GOOGLE_APPLICATION_CREDENTIALS=$(pwd -P)/gcp-credentials.json

.tests:
  stage: test
  except:
    - tags
    - schedules

trufflehog:
  stage: trufflehog
  script:
    - pip install trufflehog
    - wget -O regex.json https://raw.githubusercontent.com/HumanCellAtlas/dcplib/master/components/trufflehog_regex_patterns.json
    - trufflehog --regex --rules regex.json --entropy=False https://github.com/HumanCellAtlas/data-store.git

unit_tests:
  extends: .tests
  script:
    - make -j4 parallel_test

test_search:
  extends: .tests
  script:
    - make -j1 tests/test_search.py

test_aws_indexer:
  extends: .tests
  variables:
    DSS_UNITTEST_OPTS: "-b -v TestAWSIndexer"
  script:
    - make -j1 tests/test_indexer.py

test_gcp_indexer:
  extends: .tests
  variables:
    DSS_UNITTEST_OPTS: "-b -v TestGCPIndexer"
  script:
    - make -j1 tests/test_indexer.py

test_subscriptions:
  extends: .tests
  script:
    - make -j1 tests/test_subscriptions.py

setup_fusillade:
  stage: fusillade
  script:
    - git clone -b master https://github.com/HumanCellAtlas/dcp-fusillade.git
    - cd dcp-fusillade
    - source environment
    - cd ..
# need to re-source data-store environment in case dcp-fusillade overwrites vars (e.g. AWS_DEFAULT_REGION)
    - source environment
    - if [[ -f "environment.$CI_COMMIT_REF_NAME" ]]; then
    -   source environment.$CI_COMMIT_REF_NAME
    - fi
    - cat ./roles.json | sed "s/\${stage}/${DSS_DEPLOYMENT_STAGE}/g" > temp-roles.json
    - if [ "$DSS_DEPLOYMENT_STAGE" == "dev" ]; then
    -   FUS_STAGE='testing'
    - else
    -   FUS_STAGE=$DSS_DEPLOYMENT_STAGE
    - fi
    - python -m json.tool ./temp-roles.json > /dev/null || exit 1
    - dcp-fusillade/scripts/setup_fusillade.py --file temp-roles.json --force $FUS_STAGE
    - scripts/check_fusillade.py $FUS_STAGE
  except:
    - schedules
  only:
    - master
    - integration
    - staging

deploy:
  stage: deploy
  script:
    - make plan-infra
    - make deploy
    - scripts/set_version.sh
  environment:
    name: $CI_COMMIT_REF_NAME
    # TODO: include url when GitLab fixes it's wonky interpolation rules
    # Variable created in this file (outside of `variables`) are not interpolated,
    # causing GitLabs environment mechanism to fail silently.
    # issue: https://gitlab.com/gitlab-com/support-forum/issues/2814
    # fix: https://gitlab.com/gitlab-org/gitlab-ce/issues/27921
    # - Brian Hannafiouus
    # url: https://dss.{dev/integration/master}.data.humancellatlas.org
  only:
    - master
    - integration
    - staging
  except:
    - schedules

integration_test:
  stage: integration_test
  script:
    - make -j1 integration_test
  only:
    - master
    - integration
    - staging
  except:
    - schedules
  variables:
    DSS_UNITTEST_OPTS: "-v"

scale_and_performance:
  stage: scale_and_performance
  script:
    - echo $DSS_JOB_COMMAND
    - $DSS_JOB_COMMAND
  only:
    variables:
      - $DSS_JOB_COMMAND

.release:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - export status=$(scripts/status.py HumanCellAtlas dcp $(echo $RELEASE_COMMAND | cut -d ' ' -f 3))
    - if [[ ${RELEASE_COMMAND} != *"--force"* && ${status} != "success" ]]; then
    -   echo "DCP Integration test returned status ${status}";
    -   exit 1
    - fi
    - ${RELEASE_COMMAND}
  except:
    - schedules
  when: manual
  allow_failure: true

release_integration:
  extends: .release
  variables:
    RELEASE_COMMAND: scripts/release.sh master integration
  only:
    - master

force_release_integration:
  extends: .release
  variables:
    RELEASE_COMMAND: scripts/release.sh master integration --force
  only:
    - master

release_staging:
  extends: .release
  variables:
    RELEASE_COMMAND: scripts/release.sh integration staging
  only:
    - integration

force_release_staging:
  extends: .release
  variables:
    RELEASE_COMMAND: scripts/release.sh integration staging --force
  only:
    - integration

release_prod:
  extends: .release
  variables:
    RELEASE_COMMAND: scripts/release.sh staging prod
  only:
    - staging

force_release_prod:
  extends: .release
  variables:
    RELEASE_COMMAND: scripts/release.sh staging prod --force
  only:
    - staging