Skip to content

Commit 7a0e086

Browse files
dakotabenjamindakotabenjamin
authored
Merge pull request #6649 from hotosm/tasking-manager-fastapi
Tasking manager fastapi
2 parents c8f5dff + 564a1ab commit 7a0e086

File tree

323 files changed

+27425
-19718
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

323 files changed

+27425
-19718
lines changed

‎.github/workflows/ecs-deploy.yml

+26-22
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,23 @@ on:
44
push:
55
branches:
66
- tasking-manager-fastapi
7+
paths:
8+
- "backend/**"
9+
workflow_dispatch:
710

811
env:
912
REGISTRY: ghcr.io
1013
AWS_REGION: us-east-1
11-
ECS_CLUSTER: tasking-manager
12-
ECS_SERVICE: tasking-manager-fastAPI
13-
CONTAINER_NAME: backend
14-
IMAGE_NAME: hotosm/tasking-manager-backend # was ${{ github.repository }}
14+
IMAGE_NAME: hotosm/tasking-manager-backend
15+
TASK_DEFINITION: tasking-manager-hotosm-staging-fastapi
16+
ECS_CLUSTER: tasking-manager-staging-cluster
17+
ECS_SERVICE: tasking-manager-hotosm-staging-fastapi
18+
CONTAINER_NAME: tasking-manager-hotosm-staging-fastapi
19+
OIDC_ROLE_ARN: arn:aws:iam::670261699094:role/Github-AWS-OIDC
1520

1621
jobs:
17-
build-push-image:
18-
name: Build Images
22+
image-build-and-push:
23+
name: Build Container Images
1924
runs-on: ubuntu-latest
2025
environment: production
2126

@@ -24,14 +29,11 @@ jobs:
2429
packages: write
2530

2631
outputs:
27-
imageid: steps.build-push-image.imageid
32+
image_tags: ${{ steps.meta.outputs.tags }}
2833

2934
steps:
30-
- name: Setup QEMU
31-
uses: docker/setup-qemu-action@v3
32-
33-
- name: Setup Buildx
34-
uses: docker/setup-buildx-action@v3
35+
- uses: docker/setup-qemu-action@v3
36+
- uses: docker/setup-buildx-action@v3
3537

3638
- name: Log in to the Container registry
3739
uses: docker/login-action@v3
@@ -45,53 +47,55 @@ jobs:
4547
uses: docker/metadata-action@v5
4648
with:
4749
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
48-
4950
tags: |
50-
type=ref,event=branch
51+
type=raw,value=fastapi
5152
5253
- name: Build and push container image
5354
id: build-push-image
5455
uses: docker/build-push-action@v5
5556
with:
5657
context: "{{defaultContext}}"
58+
target: prod
5759
platforms: linux/amd64,linux/arm64
5860
push: true
5961
tags: ${{ steps.meta.outputs.tags }}
62+
labels: ${{ steps.meta.outputs.labels }}
6063

6164
deploy:
62-
name: Deploy
65+
name: Deploy to ECS
6366
runs-on: ubuntu-latest
6467
environment: production
6568

69+
needs: image-build-and-push
70+
6671
permissions:
6772
contents: read
6873
id-token: write
6974

7075
steps:
71-
- name: Checkout
72-
uses: actions/checkout@v4
76+
- uses: actions/checkout@v4
7377

7478
- name: Configure AWS credentials
7579
uses: aws-actions/configure-aws-credentials@v4
7680
with:
77-
aws-region: us-east-1
78-
role-to-assume: arn:aws:iam::670261699094:role/Github-AWS-OIDC
81+
aws-region: ${{ env.AWS_REGION }}
82+
role-to-assume: ${{ env.OIDC_ROLE_ARN }}
7983
role-session-name: gh-ci-ecs-deploy
8084

8185
- name: Download task definition
8286
run: |
83-
aws ecs describe-task-definition --task-definition tasking-manager --query taskDefinition > task-definition.json
87+
aws ecs describe-task-definition --task-definition ${{ env.TASK_DEFINITION }} --query taskDefinition > task-definition.json
8488
8589
- name: Task definition rendition
8690
id: task-def
8791
uses: aws-actions/amazon-ecs-render-task-definition@v1
8892
with:
8993
task-definition: task-definition.json
9094
container-name: ${{ env.CONTAINER_NAME }}
91-
image: ${{ needs.build-push-image.outputs.imageid }}
95+
image: ${{ needs.image-build-and-push.outputs.image_tags }}
9296

9397
- name: Deploy task definition
94-
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
98+
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
9599
with:
96100
task-definition: ${{ steps.task-def.outputs.task-definition }}
97101
service: ${{ env.ECS_SERVICE }}

‎.github/workflows/terragrunt-apply.yml

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Terragrunt Apply
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Terragrunt Plan"]
6+
types:
7+
- completed
8+
workflow_dispatch:
9+
inputs:
10+
plan_file_name:
11+
description: "Enter Terragrunt Plan File name to run"
12+
required: true
13+
14+
jobs:
15+
# workaround for using GHActions environment variables feature. In future we can use ${{ github.ref_name }} directly in the workflow as INFRA_BRANCH
16+
get_deployment_meta:
17+
name: Get Deployment Meta
18+
runs-on: ubuntu-latest
19+
outputs:
20+
PLAN_NAME: ${{ steps.export_meta.outputs.PLAN_NAME }}
21+
INFRA_BRANCH_NAME: ${{ steps.export_triggering_wf_meta.outputs.INFRA_BRANCH_NAME }}
22+
INFRA_BRANCH_URL: ${{ steps.export_triggering_wf_meta.outputs.INFRA_BRANCH_URL }}
23+
INFRA_MODULE_PATH: ${{ steps.export_triggering_wf_meta.outputs.INFRA_MODULE_PATH }}
24+
25+
steps:
26+
- name: Export Deployment Meta
27+
id: export_meta
28+
shell: bash
29+
run: |
30+
case "${{ github.event_name }}" in
31+
workflow_run)
32+
export PLAN_NAME=tasking-manager-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
33+
;;
34+
workflow_dispatch)
35+
export PLAN_NAME=${{ inputs.plan_file_name }}
36+
;;
37+
esac
38+
echo "PLAN_NAME=${PLAN_NAME}" >> $GITHUB_OUTPUT
39+
40+
- name: Get Input Params from GitHub Artifacts
41+
uses: actions/download-artifact@v4
42+
with:
43+
name: PLAN_WORKFLOW_META.info
44+
github-token: ${{ github.token }}
45+
repository: ${{ github.repository }}
46+
run-id: ${{ github.event.workflow_run.id }}
47+
48+
- name: Export triggering plan workflow meta
49+
id: export_triggering_wf_meta
50+
shell: bash
51+
run: |
52+
for line in $(cat PLAN_WORKFLOW_META);
53+
do
54+
echo $line >> $GITHUB_OUTPUT
55+
done
56+
57+
apply:
58+
name: Terragrunt Apply
59+
uses: hotosm/gh-workflows/.github/workflows/terragrunt-apply.yml@3.1.0
60+
permissions:
61+
id-token: write
62+
contents: read
63+
needs:
64+
- get_deployment_meta
65+
with:
66+
working_dir: ./scripts/aws/infra/${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}/${{ needs.get_deployment_meta.outputs.INFRA_MODULE_PATH }}
67+
terraform_version: "1.9.5"
68+
terragrunt_version: "0.67.15"
69+
aws_region: us-east-1
70+
load_env: true
71+
plan_file_name: ${{ needs.get_deployment_meta.outputs.PLAN_NAME }}
72+
use_gh_artifacts: true
73+
environment_name: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}
74+
environment_url: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_URL }}
75+
decrypt_plan_file: true
76+
secrets: inherit

‎.github/workflows/terragrunt-plan.yml

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Terragrunt Plan
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
module_path:
7+
description: "Select Terragrunt Module to run"
8+
required: true
9+
type: choice
10+
default: purgeable/ecs/
11+
options:
12+
- purgeable/ecs/
13+
- purgeable/ecs-cron/
14+
- non-purgeable/extras/
15+
16+
jobs:
17+
# workaround for using GHActions environment variables feature. In future we can use ${{ github.ref_name }} directly in the workflow as INFRA_BRANCH
18+
get_deployment_meta:
19+
name: Get Deployment Meta
20+
runs-on: ubuntu-latest
21+
outputs:
22+
INFRA_BRANCH_NAME: ${{ steps.export_meta.outputs.INFRA_BRANCH_NAME }}
23+
INFRA_BRANCH_URL: ${{ steps.export_meta.outputs.INFRA_BRANCH_URL }}
24+
steps:
25+
- name: Export Deployment Meta
26+
id: export_meta
27+
shell: bash
28+
run: |
29+
case "${{ github.ref }}" in
30+
refs/heads/develop)
31+
export INFRA_BRANCH_NAME=staging
32+
export INFRA_BRANCH_URL=https://tasks-stage.hotosm.org
33+
;;
34+
refs/heads/tasking-manager-fastapi)
35+
export INFRA_BRANCH_NAME=staging
36+
export INFRA_BRANCH_URL=https://tasks-stage.hotosm.org
37+
;;
38+
esac
39+
echo "INFRA_BRANCH_NAME=${INFRA_BRANCH_NAME}" >> $GITHUB_OUTPUT
40+
echo "INFRA_BRANCH_URL=${INFRA_BRANCH_URL}" >> $GITHUB_OUTPUT
41+
42+
- name: Write Inputs as Artifacts
43+
shell: bash
44+
run: |
45+
set -e
46+
echo INFRA_BRANCH_NAME=${{ steps.export_meta.outputs.INFRA_BRANCH_NAME }} >> PLAN_WORKFLOW_META
47+
echo INFRA_BRANCH_URL=${{ steps.export_meta.outputs.INFRA_BRANCH_URL }} >> PLAN_WORKFLOW_META
48+
echo INFRA_MODULE_PATH=${{ inputs.module_path }} >> PLAN_WORKFLOW_META
49+
50+
- name: Upload Inputs as Artifacts
51+
uses: actions/upload-artifact@v4
52+
with:
53+
name: PLAN_WORKFLOW_META.info
54+
path: ./PLAN_WORKFLOW_META
55+
retention-days: 1
56+
57+
plan:
58+
name: Terragrunt Plan
59+
uses: hotosm/gh-workflows/.github/workflows/terragrunt-plan.yml@3.1.0
60+
permissions:
61+
id-token: write
62+
contents: read
63+
needs:
64+
- get_deployment_meta
65+
with:
66+
working_dir: ./scripts/aws/infra/${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}/${{ github.event.inputs.module_path }}
67+
terraform_version: "1.9.5"
68+
terragrunt_version: "0.67.15"
69+
aws_region: us-east-1
70+
load_env: true
71+
environment_name: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}
72+
environment_url: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_URL }}
73+
encrypt_plan_file: true
74+
secrets: inherit

‎.gitignore

+10-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
# Configuration files #
32
tasking-manager*.env
43

@@ -65,6 +64,16 @@ htmlcov/
6564

6665
# Terragrunt
6766
.terragrunt-cache
67+
**/.terragrunt-cache/
68+
**/.terragrunt-cache
69+
**/infra.env
6870

6971
# Docker & Docker compose
7072
docker-compose.override.yml
73+
postgres_data/
74+
75+
# Variables
76+
scripts/aws/infra/staging/variables.py
77+
78+
# Files
79+
tests/test_db_dump/tm_sample_db.sql

‎.pre-commit-config.yaml

+52-15
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ repos:
88
entry: trailing-whitespace-fixer
99
language: python
1010
types: [text]
11-
stages: [commit, push, manual]
11+
stages: [pre-commit, pre-push, manual]
1212

1313
- id: end-of-file-fixer
1414
name: fix end of files
1515
description: ensures that a file is either empty, or ends with one newline.
1616
entry: end-of-file-fixer
1717
language: python
1818
types: [text]
19-
stages: [commit, push, manual]
19+
stages: [pre-commit, pre-push, manual]
2020

2121
- id: detect-aws-credentials
2222
name: detect aws credentials
@@ -60,7 +60,7 @@ repos:
6060
description: prevents giant files from being committed.
6161
entry: check-added-large-files
6262
language: python
63-
stages: [commit, push, manual]
63+
stages: [pre-commit, pre-push, manual]
6464
args: ['--maxkb=10240']
6565

6666
# Versioning: Commit messages & changelog
@@ -70,18 +70,18 @@ repos:
7070
- id: commitizen
7171
stages: [commit-msg]
7272

73-
# Lint / autoformat: Python code
74-
- repo: https://github.com/astral-sh/ruff-pre-commit
75-
# Ruff version.
76-
rev: "v0.6.4"
77-
hooks:
78-
# Run the linter
79-
- id: ruff
80-
files: ^backend/(?:.*/)*.*$
81-
args: [--fix, --exit-non-zero-on-fix]
82-
# Run the formatter
83-
- id: ruff-format
84-
files: ^backend/(?:.*/)*.*$
73+
# # Lint / autoformat: Python code
74+
# - repo: https://github.com/astral-sh/ruff-pre-commit
75+
# # Ruff version.
76+
# rev: "v0.6.4"
77+
# hooks:
78+
# # Run the linter
79+
# - id: ruff
80+
# files: ^backend/(?:.*/)*.*$
81+
# args: [--fix, --exit-non-zero-on-fix]
82+
# # Run the formatter
83+
# - id: ruff-format
84+
# files: ^backend/(?:.*/)*.*$
8585

8686
# INFO: Searches for code that is used or lingering around. (Disabled since there were a lot of work from dev end to remove stuff)
8787
# - repo: https://github.com/asottile/dead
@@ -104,3 +104,40 @@ repos:
104104
# "!frontend/pnpm-lock.yaml",
105105
# "!backend/tests/test_data/**",
106106
# ]
107+
108+
- repo: https://github.com/psf/black
109+
rev: 23.12.1
110+
hooks:
111+
- id: black
112+
language_version: python3.10
113+
114+
115+
- repo: https://github.com/PyCQA/flake8
116+
rev: "7.1.2"
117+
hooks:
118+
- id: flake8
119+
name: flake8
120+
additional_dependencies: [mccabe>=0.7.0]
121+
args:
122+
[
123+
"--max-line-length=119",
124+
"--max-complexity=150",
125+
"--ignore=E203,W503",
126+
"--extend-exclude=migrations/*",
127+
]
128+
files: '^(backend|tests|manage\.py)'
129+
130+
# - repo: https://github.com/psf/black
131+
# rev: "23.12.1" # Please keep this version updated, should be same as your black version
132+
# hooks:
133+
# - id: black
134+
# name: black tests
135+
# entry: black
136+
# args:
137+
# [
138+
# "--line-length=88",
139+
# "manage.py",
140+
# "backend",
141+
# "tests",
142+
# "migrations",
143+
# ]

0 commit comments

Comments
 (0)