diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 9a3703922..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,309 +0,0 @@ -version: 2.1 - -references: - .workspace_root: &workspace_root /tmp/repo - - .attach_root: &attach_root - attach_workspace: - at: *workspace_root - - .image_client: &image_client - image: cimg/node:16.14-browsers - - .working_directory_root: &working_directory_root - working_directory: *workspace_root - - .base_env: &base_env - docker: - - <<: *image_client - working_directory: *workspace_root - - .testing_branches: &testing_branches - filters: - branches: - ignore: main - tags: - ignore: /(.*)/ - - .all_branches_and_tags: &all_branches_and_tags - filters: - branches: - only: /(.*)/ - tags: - only: /^\d+\.\d+\.\d(.*)/ # semver - - .main_and_tags: &main_and_tags - filters: - branches: - only: main - tags: - only: /^\d+\.\d+\.\d(.*)/ # semver - -commands: - login_dockerhub: - steps: - - run: - name: Log into DockerHub - command: docker login -u "${QUAY_USER}" -p "${QUAY_PASS}" quay.io - build_docker_image: - parameters: - dockerfile: - type: string - steps: - - setup_remote_docker: { docker_layer_caching: true, version: 20.10.2 } - - run: - name: Build Docker image - command: | - if echo "${CIRCLE_TAG}" | grep "^[0-9]\+\.[0-9]\+\.[0-9]\+$"; then - DOCKER_BUILDKIT=1 docker build -t ${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1} --build-arg COMMIT_HASH=${CIRCLE_SHA1} --build-arg CURRENT_VERSION=${CIRCLE_TAG} -f <> . - else - DOCKER_BUILDKIT=1 docker build -t ${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1} --build-arg COMMIT_HASH=${CIRCLE_SHA1} --build-arg CURRENT_VERSION=${CIRCLE_BRANCH} -f <> . - fi - push_docker_image: - steps: - - run: - name: Push image - command: | - docker push "${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1}" - if echo "${CIRCLE_TAG}" | grep "^[0-9]\+\.[0-9]\+\.[0-9]\+$"; then - docker tag "${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1}" "${DOCKER_IMAGE_NAME}:${CIRCLE_TAG}" - docker push "${DOCKER_IMAGE_NAME}:${CIRCLE_TAG}" - docker tag "${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1}" "${DOCKER_IMAGE_NAME}:latest" - docker push "${DOCKER_IMAGE_NAME}:latest" - # TODO: publish SNS message for stable channel - fi - - if [ "${CIRCLE_BRANCH}" = "main" ]; then - docker tag "${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1}" "${DOCKER_IMAGE_NAME}:next" - docker push "${DOCKER_IMAGE_NAME}:next" - - # TODO: publish SNS message for latest channel - fi - -jobs: - checkout: - <<: *base_env - - steps: - - checkout - - # Download and cache dependencies - - restore_cache: - name: Restore Yarn Package Cache - keys: - - yarn-packages-{{ checksum "yarn.lock" }} - - - run: - name: Install Dependencies - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - - - save_cache: - name: Save Yarn Package Cache - key: yarn-packages-{{ checksum "yarn.lock" }} - paths: - - ~/.cache/yarn - - # persist to workspace to use in downstream jobs - - persist_to_workspace: - root: *workspace_root - paths: - - . - - linter: - <<: *base_env - steps: - - <<: *attach_root - - run: yarn lint:check - - build-content: - docker: - - <<: *image_client - - image: cimg/postgres:12.9 - command: postgres -c max_connections=300 - environment: - POSTGRES_DB: postgres_test - POSTGRES_PASSWORD: '12345678' - POSTGRES_USER: postgres - parallelism: 5 - working_directory: *workspace_root - steps: - - <<: *attach_root - - run: - name: Run tests - command: | - cd content - TESTFILES=$(circleci tests glob "test/**/*.spec.ts" | circleci tests split --split-by=timings) - yarn test:ci $TESTFILES --coverageDirectory=reports --reporters=jest-junit --reporters=default - mv reports/coverage-final.json reports/coverage-${CIRCLE_NODE_INDEX:-0}.json - mv api-coverage/api-coverage.csv api-coverage/api-coverage-${CIRCLE_NODE_INDEX:-0}.csv || true - environment: - CI: 'true' - API_COVERAGE: 'true' - JEST_JUNIT_OUTPUT_DIR: './test-result' - JEST_JUNIT_ADD_FILE_ATTRIBUTE: 'true' - - store_test_results: - path: content/test-result - - store_artifacts: - path: content/test-result - - persist_to_workspace: - root: *workspace_root - paths: - - content/reports - - content/api-coverage - - report-content-coverage: - <<: *base_env - steps: - - <<: *attach_root - - run: - name: Report coverage - command: | - cd content - yarn nyc merge reports - mkdir .nyc_output/ - mv coverage.json .nyc_output/coverage.json - yarn nyc report --reporter=lcov --reporter=text - - store_test_results: - path: content/coverage - - store_artifacts: - path: content/coverage - report-content-api-coverage: - <<: *base_env - steps: - - <<: *attach_root - - run: - name: Report API coverage - command: | - cd content - yarn report-api-coverage - build-lambdas: - <<: *base_env - steps: - - <<: *attach_root - - run: - name: Build content - command: yarn workspace @catalyst/lambdas-server build - - run: - name: Run tests - command: yarn workspace @catalyst/lambdas-server test - - publish-content-docker: - <<: *base_env - docker: - - image: cimg/node:16.14-browsers - environment: &ENVIRONMENT - DOCKER_IMAGE_NAME: quay.io/decentraland/catalyst-content - steps: - - <<: *attach_root - - build_docker_image: - dockerfile: "Dockerfile.content" - - login_dockerhub - - push_docker_image - - publish-lambdas-docker: - <<: *base_env - docker: - - image: cimg/node:16.14-browsers - environment: &ENVIRONMENT - DOCKER_IMAGE_NAME: quay.io/decentraland/catalyst-lambdas - steps: - - <<: *attach_root - - build_docker_image: - dockerfile: "Dockerfile.lambdas" - - login_dockerhub - - push_docker_image - - publish-content-docker-without-tag: - <<: *base_env - docker: - - image: cimg/node:16.14-browsers - environment: &ENVIRONMENT - DOCKER_IMAGE_NAME: quay.io/decentraland/catalyst-content - steps: - - <<: *attach_root - - build_docker_image: - dockerfile: "Dockerfile.content" - - login_dockerhub - - run: - name: Push image (without tag) - command: | - docker push "${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1}" - - publish-lambdas-docker-without-tag: - <<: *base_env - docker: - - image: cimg/node:16.14-browsers - environment: &ENVIRONMENT - DOCKER_IMAGE_NAME: quay.io/decentraland/catalyst-lambdas - steps: - - <<: *attach_root - - build_docker_image: - dockerfile: "Dockerfile.lambdas" - - login_dockerhub - - run: - name: Push image (without tag) - command: | - docker push "${DOCKER_IMAGE_NAME}:${CIRCLE_SHA1}" - -workflows: - version: 2 - - build-deploy: - jobs: - # checkout code - - checkout: - <<: *all_branches_and_tags - - - linter: - <<: *all_branches_and_tags - requires: - - checkout - - # build packages - - build-content: - <<: *all_branches_and_tags - requires: - - checkout - - report-content-coverage: - <<: *all_branches_and_tags - requires: - - build-content - - report-content-api-coverage: - <<: *all_branches_and_tags - requires: - - build-content - - build-lambdas: - <<: *all_branches_and_tags - requires: - - checkout - - # publish docker images in "main" and semver tags - # main publishes to @next - # semver tags publish to @latest - - publish-content-docker: - <<: *main_and_tags - requires: - - build-content - - report-content-coverage - - report-content-api-coverage - - - publish-lambdas-docker: - <<: *main_and_tags - requires: - - build-lambdas - - # NOT (main | semver) enables manual approval to release untagged docker images - - accept-publish-docker-for-testing: - <<: *testing_branches - type: approval - requires: - - build-content - - build-lambdas - - publish-content-docker-without-tag: - <<: *testing_branches - requires: - - accept-publish-docker-for-testing - - publish-lambdas-docker-without-tag: - <<: *testing_branches - requires: - - accept-publish-docker-for-testing diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..c2d389de1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,71 @@ +name: build + +on: + push: + branches: + - 'main' + pull_request: + +jobs: + validations: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 18.x + uses: actions/setup-node@v4 + with: + node-version: 18.x + cache: yarn + - name: install + run: yarn install --frozen-lockfile + - name: build + run: yarn build + - name: lint + run: yarn lint:check + + test-content: + runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_DB: postgres_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: 12345678 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v4 + - name: Use Node.js 16.16.0 + uses: actions/setup-node@v4 + with: + node-version: 16.16.0 + cache: yarn + - name: install + run: yarn install --frozen-lockfile + - name: create .env + run: echo "" >> .env + - name: test + run: yarn test:content + + + test-lambdas: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Use Node.js 16.16.0 + uses: actions/setup-node@v3 + with: + node-version: 16.16.0 + cache: yarn + - name: install + run: yarn install --frozen-lockfile + - name: create .env + run: echo "" >> .env + - name: test + run: yarn test:lambdas diff --git a/.github/workflows/docker-next.yml b/.github/workflows/docker-next.yml new file mode 100644 index 000000000..f05f5fe12 --- /dev/null +++ b/.github/workflows/docker-next.yml @@ -0,0 +1,13 @@ +name: CI/CD on main branch + +on: + push: + branches: + - "main" + +jobs: + cd: + uses: decentraland/platform-actions/.github/workflows/apps-docker-next.yml@main + with: + service-name: catalyst + secrets: inherit diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 000000000..b74ff6754 --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,13 @@ +name: Publish 'latest' image + +on: + release: + types: + - "created" + +jobs: + cd: + uses: decentraland/platform-actions/.github/workflows/apps-docker-release.yml@main + with: + service-name: catalyst + secrets: inherit diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 000000000..7ecea6f85 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,11 @@ +name: CI/CD on PR + +on: + pull_request: + +jobs: + pr: + uses: decentraland/platform-actions/.github/workflows/apps-pr.yml@main + with: + service-name: catalyst + secrets: inherit diff --git a/Dockerfile.content b/Dockerfile similarity index 83% rename from Dockerfile.content rename to Dockerfile index 6ce87d978..6f0ec1aa9 100644 --- a/Dockerfile.content +++ b/Dockerfile @@ -4,8 +4,9 @@ RUN apk add --no-cache bash git COPY package.json . COPY yarn.lock . -COPY blocks-cache-*.csv . +COPY content/blocks-cache-*.csv content/ COPY content/package.json content/ +COPY lambdas/package.json lambdas/ # get production dependencies FROM base as dependencies @@ -14,10 +15,8 @@ RUN yarn install --prod --frozen-lockfile # build sources FROM base as catalyst-builder RUN yarn install --frozen-lockfile - COPY . . -FROM catalyst-builder as content-builder -RUN yarn workspace @catalyst/content-server build +RUN yarn build # build final image with transpiled code and runtime dependencies FROM base @@ -25,8 +24,9 @@ FROM base COPY --from=dependencies /app/node_modules ./node_modules/ COPY --from=dependencies /app/content/node_modules ./node_modules/ -COPY --from=content-builder /app/content/dist/src content/ -COPY --from=content-builder /app/content/blocks-cache-*.csv . +COPY --from=catalyst-builder /app/content/dist/src content/ +COPY --from=catalyst-builder /app/content/blocks-cache-*.csv /app/ +COPY --from=catalyst-builder /app/lambdas/dist/src lambdas/ # https://docs.docker.com/engine/reference/builder/#arg ARG CURRENT_VERSION=4.0.0-ci diff --git a/Dockerfile.lambdas b/Dockerfile.lambdas deleted file mode 100644 index 47b9f3c0e..000000000 --- a/Dockerfile.lambdas +++ /dev/null @@ -1,49 +0,0 @@ -FROM node:16-alpine as base - -WORKDIR /app -RUN apk add --no-cache bash git - -COPY package.json . -COPY yarn.lock . -COPY lambdas/package.json lambdas/ - -# get production dependencies -FROM base as dependencies -RUN yarn install --prod --frozen-lockfile - -# build sources -FROM base as catalyst-builder -RUN yarn install --frozen-lockfile - -COPY . . -FROM catalyst-builder as lambdas-builder -RUN yarn workspace @catalyst/lambdas-server build - -# build final image with transpiled code and runtime dependencies -FROM base - -COPY --from=dependencies /app/node_modules ./node_modules/ -COPY --from=lambdas-builder /app/lambdas/dist/src lambdas/ - -# https://docs.docker.com/engine/reference/builder/#arg -ARG CURRENT_VERSION=4.0.0-ci -ENV CURRENT_VERSION=${CURRENT_VERSION:-4.0.0} - -# https://docs.docker.com/engine/reference/builder/#arg -ARG COMMIT_HASH=local -ENV COMMIT_HASH=${COMMIT_HASH:-local} - -EXPOSE 7070 - -# Please _DO NOT_ use a custom ENTRYPOINT because it may prevent signals -# (i.e. SIGTERM) to reach the service -# Read more here: https://aws.amazon.com/blogs/containers/graceful-shutdowns-with-ecs/ -# and: https://www.ctl.io/developers/blog/post/gracefully-stopping-docker-containers/ - -# We use Tini to handle signals and PID1 (https://github.com/krallin/tini, read why here https://github.com/krallin/tini/issues/8) -RUN apk add --no-cache tini - -ENTRYPOINT ["/sbin/tini", "--"] - -# Run the program under Tini -CMD [ "/usr/local/bin/node", "--max-old-space-size=8192", "lambdas/entrypoints/run-server.js" ] diff --git a/content/src/service/synchronization/retryFailedDeployments.ts b/content/src/service/synchronization/retryFailedDeployments.ts index d7c9b19b3..0d2d1a587 100644 --- a/content/src/service/synchronization/retryFailedDeployments.ts +++ b/content/src/service/synchronization/retryFailedDeployments.ts @@ -27,7 +27,8 @@ export const createRetryFailedDeployments = ( ): IRetryFailedDeploymentsComponent => { const retryDelay = components.env.getConfig(EnvironmentConfig.RETRY_FAILED_DEPLOYMENTS_DELAY_TIME) const logger = components.logs.getLogger('RetryFailedDeployments') - const ac = new AbortController() + let ac: AbortController | undefined = new AbortController() + const signal = ac.signal let running = false return { @@ -37,12 +38,15 @@ export const createRetryFailedDeployments = ( }, stop: async () => { running = false - ac.abort() + if (ac) { + ac.abort() + } + ac = undefined logger.debug('Stopping retry failed deployments') }, schedule: async () => { while (running) { - await setTimeout(retryDelay, null, { signal: ac.signal }) + await setTimeout(retryDelay, null, { signal }) if (!running) { return } diff --git a/package.json b/package.json index 1ad5160fe..08e98ef54 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "start": "yarn workspaces run start:server", "start:content": "yarn workspace @catalyst/content-server start:db && yarn workspace @catalyst/content-server start:server", "start:lambdas": "yarn workspace @catalyst/lambdas-server start:server", - "test": "run-s 'test:*'", + "test": "yarn test:content && yarn test:lambdas", "test:content": "yarn workspace @catalyst/content-server test", "test:lambdas": "yarn workspace @catalyst/lambdas-server test", "unused": "npx depcheck; yarn unused:content; yarn unused:lambdas",