Initial docker workflow

.dockerignore

@@ -0,0 +1,29 @@
+# Dynalite
+*.out
+
+# OSX
+.DS_Store
+
+# doit
+.doit.db
+
+# Virtual Environment
+venv
+
+# Eggs
+.eggs
+subhub.egg-info
+
+# Node
+node_modules
+
+# Visual Studio Code
+.vscode
+
+# PyCharm
+.idea
+
+.vscode/
+.tox
+venv
+.doit.db

.travis.yml

@@ -2,6 +2,8 @@ language: python
 dist: xenial
 python:
   - "3.7"
+services:
+  - docker
 env:
   global:
     - AWS_DEFAULT_REGION: us-west-2
@@ -10,8 +12,6 @@ env:
     - USER_TABLE: testing
     - DEPLOY_ENV: dev
     - secure: ldfVKFlO/kQLAmXFTf+UIsj1OzGV+EtfF1mEqp9ArD9eZg0EiM6mr2LTWMAKUyUlD3uCXSkF0xmIUIxNSUB+FwDxSNZdtO+Fx3hmF6I6ofX2NGEDdz2l/F7LeC3MfyPU7qwL9IL15gX2w6got78eDtdGMlhmt2s8dNsWDbLlAMjGZtCvfcXjY5GJS96Pe4lNHbCNlAi+x8E2UsyOBAEAySXfCfbS1JLW9XqiBKk1hinE+f2j47Wb6DJVX3JKfYLgdRdO+4JVa5dJvBIGhmO3lTi+vwDpTx+QbI+/oaoLpqSBpPwcpycZSPCJ0Oo+HfR8DCQRmT8NB6uc/wD44hrYlOXfXudaUH1TzQ2eT+ZT6BY5ikCArHpRDFHzlu2XmbTRq+phzArlV54H8xd46sIrMkLlVP4UHQ1GVOnXWTlG6wkrXxyPUFPAFeSKOiiK2UoHgz6j7yOTwKkuYH3/96GzH+YmoNZPFaDCh31YDFW4+ynlHFWA96U9+GKVS66f52aNMf7El6SwYSeuOzfOnKo5h5opKkkjO83KVrfzwBwpYkXHtyMG2o8hhJUDBF6VozPusJllkZfdyapdQgdTtH5R4Es1QNkzLAfX/iNGfr1yMc+I6O7VLEoB30202m1GDaocqnUwCHnMhnPdkw8E1x0DXoqSBHlCr/GJMjNiRsPjZ0k=
-    - secure: eIOTi2ri6Y07/lqulILb6wQDC3SiZkiEPALG22icnOMQJJc7iSQCatyxZAze4M1aZRoQ3RgvKcftLFFB55JtXKwAoeQMU2eLSls31weKU4WgBqDwItJaKbbeDM/tBUYKel1rocx1rjxAGB/l4/YavlLwVyVOuA3MZ/bRbBEyBWsTJsdSpSUmEjXmWEcrk4ielMxZYBjbvSpa++FDDQvxSPzt5xeu66fa885i1u95MSNuaCJa3zkcueopEu1k/PA/IpqBr4Ga5llKer3epnByyy35JOnfBC4GC/Pu4B/520NaAj1YFP2Fuz9ZifSm9+E3iwHV1Ui87Xft8c3Qu0nIev9n9s3VUIq40Q5nWTmL9lolULdnQdWW5spHyKZGZKO4lX66KNe7Pj/B+58FUNYTzNkxhM1K1361QDrF9isgNgE/yCIcM4jeDHuZcnw0yjJe5b5Rv/RMIaEL/tz7DUSFKIWxbS6ESBVSyBoyxcph9b0mWH1LIKTWtFbFMS35Fxt962ozPmK/Wq+SXOfdPk7Um4Jz3o3/T12hFfcaqwXkeMtqRioq+m67+tYdWRUcflVyrXP4c3z5obLlo3t3bWRKSrr3ca5j60yxt+wbD+jt53A4hR9jrlRXsiMlm4T0GOE10loF2rYIzrnL+2yOD8cpaKG0StqLh4IUYeJk4X7WtlY=
-    - secure: SGLE6u5bXFchF9jCtiiuYkp3Eyz7ghwyS7TvHcfv6SBsA1QGXPyVUsU9QcoLB3srajwvponYOLAyynD+tG9AdqgvmueRpUh+h3qckU1NPfDUEefkJ9zdYcT+n7TCGBYePRuYUbLMtRvX8NDUb9LTD3U+u80L53UjFfTrRQH8csE4tfOiqpPrzbKAMBnrl7fyBWHCwjkICoce01wEEp2+T4b6Sq/qWUkKrPn6KBnHO3Ev+WtEPGcF9pMrhdwkV8vsA8C9KYffDbT7s3dAlZIhEtIhD1komqMYdfTuokVyRlrWqcG4g++7/ZPGYCtFTpPx8o3qQfoLhgKMe7oOs+WivqmFtsOV6JywDDR4brBQI3lUtRK2yIrBhdhC/BAogol8g1A5asPL2uGPJtrc4O4E1x5mGgDJLtpefM3zWOqdXiL6YGz0MYZA3hgZxLj2uVHj97pzxPZbZgyrwO6LtH0hbwTxF1L9/Ha42jPEtnjmu6fvygSaQXtjot6saCSIMZBGwYWmk5ckcZ3zgI+G9R9G0eIRwap/mKElmgQX46BqsC/npQWHTQNcv5UmMtFAFRdT76gnWS9NanvOZO3mxreo9q+cyt7yz7TS2bZpNesU1GqLF6mmyWjvJP4FsEB1JhzW+yfAK8+1xnIcRxw8n/XA6x5ZAiJLD5+2M4ukucwj8aM=
 
 before_install:
   - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL);
@@ -23,13 +23,15 @@ before_install:
   - npm install -g dynalite
   - npm install -g kinesalite
   - sudo apt-get install -y graphviz graphviz-dev awscli
-  - "./scripts/aws_credentials.sh"
+  - "./bin/aws-credentials.sh"
 
 stages:
   - name: Graph
   - name: Unit Test
-  - name: Deploy to development
-    if: branch = master
+<<<<<<< HEAD
+  - name: Docker Build
+=======
+>>>>>>> master
 
 jobs:
   include:
@@ -39,7 +41,11 @@ jobs:
         - doit draw
     - stage: Unit Test
       script:
+<<<<<<< HEAD
         - doit test
-    - stage: Deploy to development
+    - stage: Docker Build
       script:
-        - doit deploy
+        - docker-compose build
+=======
+        - doit test
+>>>>>>> master

Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.7-alpine
+MAINTAINER Stewart Henderson <shenderson@mozilla.com>
+
+ARG STRIPE_API_KEY
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG LOCAL_FLASK_PORT
+ARG SUPPORT_API_KEY
+
+ENV STRIPE_API_KEY=$STRIPE_API_KEY
+ENV AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+ENV LOCAL_FLASK_PORT=$LOCAL_FLASK_PORT
+ENV SUPPORT_API_KEY=$SUPPORT_API_KEY
+
+ENV FLASK_ENV=development
+EXPOSE $LOCAL_FLASK_PORT
+
+RUN mkdir -p /app
+WORKDIR /app
+COPY . /app
+
+RUN apk add bash==5.0.0-r0
+RUN bin/install-packages.sh
+RUN pip install -r automation_requirements.txt
+RUN pip install awscli==1.16.213
+RUN yarn install
+
+RUN addgroup -g 10001 app && \
+    adduser -D -G app -h /app -u 10001 app
+
+USER app
+ENTRYPOINT ["doit", "local"]

README.md

@@ -189,6 +189,14 @@ This command will generate a PNG of the dependency graph.
 doit draw
 ```
 
+### Docker
+* build: `docker build -t mozilla/subhub .`
+* run: `docker run -it mozilla/subhub`
+
+### Docker Compose
+* build and run: `docker-compose up --build`
+* run: `docker-compose up`
+
 ## Postman
 
 A [Postman](https://www.getpostman.com/) URL collection is available for testing, learning,

bin/alpine-packages

@@ -0,0 +1,18 @@
+build-base
+libgit2-dev
+libc-dev
+python3-dev
+libffi-dev
+openssl-dev
+zeromq-dev
+linux-headers
+nodejs
+curl
+yarn
+gcc==8.3.0-r0
+g++
+musl-dev
+pkgconfig
+git
+graphviz-dev
+graphviz

bin/aws-credentials.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# NOTE: This script is used to provision both TravisCI and Jenkins, AWS credentials and configuration
+# Reference AWS Environment Variables
+#   https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
+
+mkdir -p ~/.aws
+
+cat > ~/.aws/credentials << EOL
+[default]
+aws_access_key_id = ${AWS_ACCESS_KEY_ID:-fake-id}
+aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY:-fake-key}
+EOL
+
+cat >~/.aws/config <<-EOF
+[default]
+output=json
+region=${AWS_DEFAULT_REGION:-us-west-2}
+EOF

bin/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# NOTE: This bash script is used as an entry point from this application's docker image.
+
+sudo su
+
+printenv
+doit venv
+#/app/venv/bin/python -m setup develop
+doit local

bin/install-packages.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Alpine Registry for package versions, https://pkgs.alpinelinux.org/packages
+apk update
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+IFS=$'\n' read -d '' -r -a lines < "${DIR}/alpine-packages"
+apk add "${lines[@]}"

docker-compose.yml

@@ -0,0 +1,17 @@
+version: "3.7"
+
+services:
+  subhub:
+    container_name: subhub
+    build:
+      context: .
+      args:
+        AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
+        AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+        STRIPE_API_KEY: "sk_test_123"
+        SUPPORT_API_KEY: "support_test"
+        # NOTE: This enables debug logging of the Stripe API when running in docker-compose.
+        STRIPE_LOG: "debug"
+        LOCAL_FLASK_PORT: 5000
+    ports:
+      - "5000:5000"

dodo.py

@@ -716,9 +716,10 @@ def task_draw():
     }
 
 if __name__ == '__main__':
-    cmd = 'sudo python3 -m pip install -r automation_requirements.txt'
-    answer = input(cmd + '[Y/n] ')
-    if answer in ('', 'Y', 'y', 'Yes', 'YES', 'yes'):
-        os.system(cmd)
-    else:
-        print('automation_requirements.txt NOT installed!')
+    cmd = 'python -m pip install -r automation_requirements.txt'
+    # answer = input(cmd + '[Y/n]')
+    os.system(cmd)
+    # if answer in ('', 'Y', 'y', 'Yes', 'YES', 'yes'):
+    #     os.system(cmd)
+    # else:
+    #     print('automation_requirements.txt NOT installed!')

package.json

@@ -6,12 +6,12 @@
   },
   "license": "MPL-2.0",
   "devDependencies": {
-    "dynalite": "^2.2.0",
-    "serverless": "^1.27.3",
+    "dynalite": "2.3.2",
+    "serverless": "1.49.0",
     "serverless-dynamodb-local": "0.2.38",
-    "serverless-domain-manager": "^3.2.1",
-    "serverless-offline": "5.0.1",
-    "serverless-plugin-tracing": "^2.0.0",
+    "serverless-domain-manager": "3.2.6",
+    "serverless-offline": "5.9.0",
+    "serverless-plugin-tracing": "2.0.0",
     "serverless-python-requirements": "4.3.0"
   }
 }

services/fxa/serverless.yml

@@ -46,6 +46,8 @@ provider:
       Ref: 'Users'
     EVENT_TABLE:
       Ref: 'Events'
+    DELETED_USER_TABLE:
+      Ref: 'DeletedUsers'
   tags:
     cost-center: 1440
     project-name: subhub
@@ -77,6 +79,7 @@ provider:
       Resource:
         - { 'Fn::GetAtt': ['Users', 'Arn'] }
         - { 'Fn::GetAtt': ['Events', 'Arn'] }
+        - { 'Fn::GetAtt': ['DeletedUsers', 'Arn']}
     - Effect: Allow
       Action:
         - 'secretsmanager:GetSecretValue'
@@ -249,6 +252,18 @@ resources:
         BillingMode: PAY_PER_REQUEST
         PointInTimeRecoverySpecification:
           PointInTimeRecoveryEnabled: true
+    DeletedUsers:
+      Type: 'AWS::DynamoDB::Table'
+      Properties:
+        AttributeDefinitions:
+          - AttributeName: user_id
+            AttributeType: S
+        KeySchema:
+          - AttributeName: user_id
+            KeyType: HASH
+        BillingMode: PAY_PER_REQUEST
+        PointInTimeRecoverySpecification:
+          PointInTimeRecoveryEnabled: true
     Events:
       Type: 'AWS::DynamoDB::Table'
       Properties:
@@ -277,4 +292,7 @@ resources:
         Ref: Users
     Events:
       Value:
-        Ref: Events
+        Ref: Events
+    DeletedUsers:
+      Value:
+        Ref: DeletedUsers

services/missing-events/subhub

@@ -1 +1 @@
-/Users/mballard/PycharmProjects/subhub/subhub
+../../subhub

subhub/app.py

@@ -18,7 +18,7 @@
 from subhub import secrets
 from subhub.cfg import CFG
 from subhub.exceptions import SubHubError
-from subhub.db import SubHubAccount, HubEvent
+from subhub.db import SubHubAccount, HubEvent, SubHubDeletedAccount
 
 from subhub.log import get_logger
 
@@ -74,6 +74,9 @@ def create_app(config=None):
         table_name=CFG.USER_TABLE, region=region, host=host
     )
     app.app.hub_table = HubEvent(table_name=CFG.EVENT_TABLE, region=region, host=host)
+    app.app.subhub_deleted_users = SubHubDeletedAccount(
+        table_name=CFG.DELETED_USER_TABLE, region=region, host=host
+    )
     if not app.app.subhub_account.model.exists():
         app.app.subhub_account.model.create_table(
             read_capacity_units=1, write_capacity_units=1, wait=True
@@ -82,6 +85,10 @@ def create_app(config=None):
         app.app.hub_table.model.create_table(
             read_capacity_units=1, write_capacity_units=1, wait=True
         )
+    if not app.app.subhub_deleted_users.model.exists():
+        app.app.subhub_deleted_users.model.create_table(
+            read_capacity_units=1, write_capacity_units=1, wait=True
+        )
 
     # Setup error handlers
     @app.app.errorhandler(SubHubError)
@@ -116,6 +123,7 @@ def display_subhub_errors(e: SubHubError):
     def before_request():
         g.subhub_account = current_app.subhub_account
         g.hub_table = current_app.hub_table
+        g.subhub_deleted_users = current_app.subhub_deleted_users
         g.app_system_id = None
         if CFG.PROFILING_ENABLED:
             if "profile" in request.args and not hasattr(sys, "_called_from_test"):

subhub/cfg.py

@@ -240,6 +240,13 @@ def USER_TABLE(self):
         """
         return self("USER_TABLE", "users-testing")
 
+    @property
+    def DELETED_USER_TABLE(self):
+        """
+        DELETED_USER_TABLE
+        """
+        return self("DELETED_USER_TABLE", "deleted-users-testing")
+
     @property
     def EVENT_TABLE(self):
         """

subhub/customer.py

@@ -96,9 +96,6 @@ def fetch_customer(subhub_account: SubHubAccount, user_id: str) -> Customer:
     db_account = subhub_account.get_user(user_id)
     if db_account:
         customer = Customer.retrieve(db_account.cust_id)
-        if "deleted" in customer and customer["deleted"]:
-            subhub_account.remove_from_db(user_id)
-            customer = None
     return customer