From 500539fc5f620bb4acb0484340892bfd4ef3f801 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Mon, 5 Nov 2018 17:05:09 -0500 Subject: [PATCH 1/5] Use AWS instead of Docker --- Makefile | 30 ++++++++++++++++-------------- distributed/Dockerfile.celery | 2 +- distributed/Dockerfile.flask | 2 +- distributed/README.md | 30 ++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 distributed/README.md diff --git a/Makefile b/Makefile index e531808a..685e3ada 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,20 @@ OSPC_ANACONDA_TOKEN := `cat ~/.ospc_anaconda_token` -NEW_RELIC_TOKEN := `cat ~/.newrelic-$(VERSION)` +NEW_RELIC_TOKEN := `cat ~/.newrelic-$(MODE)` dist-build: - cd distributed && \ - docker build -t opensourcepolicycenter/distributed:$(TAG) ./ --build-arg PUF_TOKEN=$(OSPC_ANACONDA_TOKEN) && \ - docker build --no-cache --build-arg TAG=$(TAG) -t opensourcepolicycenter/flask:$(TAG) --file Dockerfile.flask ./ && \ - docker build --no-cache --build-arg TAG=$(TAG) -t opensourcepolicycenter/celery:$(TAG) --file Dockerfile.celery ./ + cd distributed && \ + docker build -t distributed:$(TAG) ./ --build-arg PUF_TOKEN=$(OSPC_ANACONDA_TOKEN) && \ + docker build --no-cache --build-arg TAG=$(TAG) -t flask:$(TAG) --file Dockerfile.flask ./ && \ + docker build --no-cache --build-arg TAG=$(TAG) -t celery:$(TAG) --file Dockerfile.celery ./ dist-push: - cd distributed && \ - docker push opensourcepolicycenter/distributed:$(TAG) && \ - docker push opensourcepolicycenter/flask:$(TAG) && \ - docker push opensourcepolicycenter/celery:$(TAG) + cd distributed && \ + docker tag distributed:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/distributed:$(TAG) && \ + docker tag flask:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/flask:$(TAG) && \ + docker tag celery:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/celery:$(TAG) && \ + docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/distributed:$(TAG) && \ + docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/flask:$(TAG) && \ + docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/celery:$(TAG) dist-test: cd distributed && \ @@ -20,12 +23,11 @@ dist-test: docker-compose rm -f webapp-build: - docker build --build-arg NEW_RELIC_TOKEN=$(NEW_RELIC_TOKEN) -t opensourcepolicycenter/web:$(TAG) ./ + docker build --build-arg NEW_RELIC_TOKEN=$(NEW_RELIC_TOKEN) -t web:$(TAG) ./ webapp-push: - docker tag opensourcepolicycenter/web:$(TAG) registry.heroku.com/ospc-$(VERSION)/web - docker push registry.heroku.com/ospc-$(VERSION)/web - docker push opensourcepolicycenter/web:$(TAG) + docker tag ospc-web:$(TAG) registry.heroku.com/ospc-$(MODE)/web + docker push registry.heroku.com/ospc-$(MODE)/web webapp-release: - heroku container:release web -a ospc-$(VERSION) + heroku container:release web -a ospc-$(MODE) diff --git a/distributed/Dockerfile.celery b/distributed/Dockerfile.celery index 943248ce..90bc72e5 100644 --- a/distributed/Dockerfile.celery +++ b/distributed/Dockerfile.celery @@ -1,5 +1,5 @@ ARG TAG -FROM opensourcepolicycenter/distributed:$TAG +FROM distributed:$TAG ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 diff --git a/distributed/Dockerfile.flask b/distributed/Dockerfile.flask index b04ed533..0d904b73 100644 --- a/distributed/Dockerfile.flask +++ b/distributed/Dockerfile.flask @@ -1,5 +1,5 @@ ARG TAG -FROM opensourcepolicycenter/distributed:$TAG +FROM distributed:$TAG LABEL build="flask" date="2018-06-13" diff --git a/distributed/README.md b/distributed/README.md new file mode 100644 index 00000000..25a174d1 --- /dev/null +++ b/distributed/README.md @@ -0,0 +1,30 @@ +# Distributed deploy instructions + +1. Build and push images to AWS container registry + ``` + export MODE= TAG= AWS_ACCOUNT_ID= AWS_REGION= + rm -rf PolicyBrain + git clone https://github.com/OpenSourcePolicyCenter/PolicyBrain + cd PolicyBrain + git fetch origin + git checkout -b $TAG $TAG + + pushd .. && make dist-build -e TAG=$TAG && \ + make dist-push -e TAG=$TAG && popd &&\ + ``` + + +2. Pull distributed images to AWS server + ``` + sudo docker-compose down + sudo docker-compose rm + docker system prune --all + sudo apt-get install awscli + sudo apt-get install python3-pip + pip3 install --upgrade awscli + aws configure + sudo $AWS_ACCOUNT_ID= $AWS_REGION= + sudo $(aws ecr get-login --region $AWS_REGION --no-include-email) + sudo REPO=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com TAG=$TAG + sudo docker-compose up -d + ``` From 2571e3e7f0ffc7b8b81dadac1b8c9862f2a47e62 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Mon, 5 Nov 2018 17:08:09 -0500 Subject: [PATCH 2/5] Parametrize docker-compose repo --- distributed/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distributed/docker-compose.yml b/distributed/docker-compose.yml index 92bbd7b3..20aeeca0 100644 --- a/distributed/docker-compose.yml +++ b/distributed/docker-compose.yml @@ -1,14 +1,14 @@ version: '3' services: flask: - image: "opensourcepolicycenter/flask:${TAG}" + image: "${REPO}/flask:${TAG}" ports: - 5050:5050 depends_on: - redis - celery celery: - image: "opensourcepolicycenter/celery:${TAG}" + image: "${REPO}/celery:${TAG}" depends_on: - redis redis: From fe8b2d2928ba55df00adfdcbe2626592e9f5f1bb Mon Sep 17 00:00:00 2001 From: hdoupe Date: Mon, 5 Nov 2018 17:09:01 -0500 Subject: [PATCH 3/5] fix spacing --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 685e3ada..aa741d45 100644 --- a/Makefile +++ b/Makefile @@ -2,19 +2,19 @@ OSPC_ANACONDA_TOKEN := `cat ~/.ospc_anaconda_token` NEW_RELIC_TOKEN := `cat ~/.newrelic-$(MODE)` dist-build: - cd distributed && \ - docker build -t distributed:$(TAG) ./ --build-arg PUF_TOKEN=$(OSPC_ANACONDA_TOKEN) && \ - docker build --no-cache --build-arg TAG=$(TAG) -t flask:$(TAG) --file Dockerfile.flask ./ && \ - docker build --no-cache --build-arg TAG=$(TAG) -t celery:$(TAG) --file Dockerfile.celery ./ + cd distributed && \ + docker build -t distributed:$(TAG) ./ --build-arg PUF_TOKEN=$(OSPC_ANACONDA_TOKEN) && \ + docker build --no-cache --build-arg TAG=$(TAG) -t flask:$(TAG) --file Dockerfile.flask ./ && \ + docker build --no-cache --build-arg TAG=$(TAG) -t celery:$(TAG) --file Dockerfile.celery ./ dist-push: - cd distributed && \ - docker tag distributed:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/distributed:$(TAG) && \ - docker tag flask:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/flask:$(TAG) && \ - docker tag celery:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/celery:$(TAG) && \ - docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/distributed:$(TAG) && \ - docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/flask:$(TAG) && \ - docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/celery:$(TAG) + cd distributed && \ + docker tag distributed:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/distributed:$(TAG) && \ + docker tag flask:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/flask:$(TAG) && \ + docker tag celery:$(TAG) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/celery:$(TAG) && \ + docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/distributed:$(TAG) && \ + docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/flask:$(TAG) && \ + docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/celery:$(TAG) dist-test: cd distributed && \ From 5d8933c01be2ad6c1d367eaed5bfc83061a1e648 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Mon, 5 Nov 2018 17:12:24 -0500 Subject: [PATCH 4/5] update RELEASES.md --- RELEASES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASES.md b/RELEASES.md index c38a18f5..9ee84440 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,7 @@ Release 1.7.4 on 2018-11-06 - None **Minor Changes** +- [#942](https://github.com/ospc-org/ospc.org/pull/939) - Use AWS Container Registry instead of DockerHub #942 - Hank Doupe - [#939](https://github.com/ospc-org/ospc.org/pull/939) - Update to Tax-Calculator 0.22.2 - Hank Doupe **Bug Fixes** From b6cb60a8f96fce7d124c0ee617619993edd87ddf Mon Sep 17 00:00:00 2001 From: hdoupe Date: Mon, 5 Nov 2018 17:19:53 -0500 Subject: [PATCH 5/5] clean up distributed deploy instructions --- distributed/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/distributed/README.md b/distributed/README.md index 25a174d1..f975c3f7 100644 --- a/distributed/README.md +++ b/distributed/README.md @@ -2,15 +2,14 @@ 1. Build and push images to AWS container registry ``` - export MODE= TAG= AWS_ACCOUNT_ID= AWS_REGION= + export TAG= AWS_ACCOUNT_ID= AWS_REGION= rm -rf PolicyBrain git clone https://github.com/OpenSourcePolicyCenter/PolicyBrain cd PolicyBrain git fetch origin git checkout -b $TAG $TAG - pushd .. && make dist-build -e TAG=$TAG && \ - make dist-push -e TAG=$TAG && popd &&\ + make dist-build && make dist-push ```