NPM cache: migrate ddb to s3

.editorconfig

@@ -0,0 +1,7 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true

.gitignore

@@ -3,3 +3,4 @@
 /coverage
 docs
 node_modules
+.vscode

README.md

@@ -86,12 +86,14 @@ Requests for a previously retrieved package are served solely from the cache, so
 
 The schedule can be adjusted in `serverless.js` (search for `rate(24 hours)`), and the update process can be triggered at any time by invoking the AutoUpdate functions.
 
-### Cached Versions
+### Cached PIP Versions
 
 The DynamoDB table for each repository stores an item for each cached package. New upstream entries will be automatically added daily (see "Automatic Updates")
 
-#### Displaying Cached NPM Versions
+#### Cached NPM Versions
 
-The NPM DynamoDB data is stored zlib compressed. It can be viewed via any zlib inflate mechanism, e.g.:
-* http://www.unit-conversion.info/texttools/compress/ (select "Decompress" in the Convert menu)
-* On the command line: `echo 'COMPRESSEDDDBDATAHERE' | base64 -d | python -c "import zlib,sys;sys.stdout.write(zlib.decompress(sys.stdin.read()))"`
+The NPM data is stored gzip format. It can be downloaded from s3bucket://${packageName}/index.gz.
+- Download the file and from the GUI click on "Decompress" in the Convert menu.
+- From the command line type `gzip -kd index.gz`
+
+New upstream entries will be automatically added daily (see "Automatic Updates")

serverless.js

@@ -58,25 +58,6 @@ async function getConfig() {
     "Type": "AWS::S3::Bucket",
     "DeletionPolicy": "Retain"
   }
-  const npmTableResourceName = "NpmTable";
-  npmResources[npmTableResourceName] = {
-    "Type": "AWS::DynamoDB::Table",
-    "Properties": {
-      "AttributeDefinitions": [
-        {
-          "AttributeName": "PackageName",
-          "AttributeType": "S"
-        }
-      ],
-      "KeySchema": [
-        {
-          "AttributeName": "PackageName",
-          "KeyType": "HASH"
-        }
-      ],
-      "BillingMode": "PAY_PER_REQUEST"
-    }
-  };
 
   const npmFunctions = {
     "npmHyphen": {
@@ -95,21 +76,41 @@ async function getConfig() {
     "npmPackage": {
       "handler": "src/npm/package/package.handler",
       "environment": {
-        "DDB_TABLE": {
-          "Ref": npmTableResourceName
-        }
+        "BUCKET_NAME": {
+          "Ref": npmStorageBucketResourceName
+        },
+        "NPM_CACHE_DOWNLOAD_URI": {
+          "Fn::Join": [
+            "",
+            ["https://",
+             {"Ref": "ApiGatewayRestApi"},
+             ".execute-api.",
+             {"Ref": "AWS::Region"},
+             ".",
+             {"Ref": "AWS::URLSuffix"},
+             "/${opt:stage}/npm-dlredirect"]
+          ]
+        }        
       },
       "iamRoleStatements": [
         {
           "Action": [
-            "dynamodb:GetItem",
-            "dynamodb:PutItem"
+            "s3:GetObject",
+            "s3:PutObject"
           ],
           "Effect": "Allow",
           "Resource": {
-            "Fn::GetAtt": [
-              npmTableResourceName,
-              "Arn"
+            "Fn::Join": [
+              "",
+              [
+                {
+                  "Fn::GetAtt": [
+                    npmStorageBucketResourceName,
+                    "Arn"
+                  ]
+                },
+                "/*"
+              ]
             ]
           }
         }
@@ -129,8 +130,8 @@ async function getConfig() {
       "handler": "src/npm/autoupdate/autoupdate.handler",
       "timeout": 900,
       "environment": {
-        "DDB_TABLE": {
-          "Ref": npmTableResourceName
+        "BUCKET_NAME": {
+          "Ref": npmStorageBucketResourceName
         },
         "NPM_CACHE_DOWNLOAD_URI": {
           "Fn::Join": [
@@ -148,16 +149,36 @@ async function getConfig() {
       "iamRoleStatements": [
         {
           "Action": [
-            "dynamodb:PutItem",
-            "dynamodb:Scan"
+            "s3:ListBucket"
           ],
           "Effect": "Allow",
           "Resource": {
             "Fn::GetAtt": [
-              npmTableResourceName,
+              npmStorageBucketResourceName,
               "Arn"
             ]
           }
+        },
+        {
+          "Action": [
+            "s3:GetObject",
+            "s3:PutObject"
+          ],
+          "Effect": "Allow",
+          "Resource": {
+            "Fn::Join": [
+              "",
+              [
+                {
+                  "Fn::GetAtt": [
+                    npmStorageBucketResourceName,
+                    "Arn"
+                  ]
+                },
+                "/*"
+              ]
+            ]
+          }
         }
       ],
       "events": [

src/npm/NpmApp.ts

@@ -0,0 +1,23 @@
+import { S3 } from "aws-sdk";
+import { S3Cache } from "./S3Cache";
+import { NpmPackage } from "./NpmPackage";
+
+const BUCKET_NAME: string = process.env.BUCKET_NAME!;
+const NPM_CACHE_DOWNLOAD_URI = process.env.NPM_CACHE_DOWNLOAD_URI!;
+
+/*
+ * Entry point to access any service of the NPM artifact cacher.
+ * It creates all instances and provides a single point of access
+ * to lambda handlers.
+ */
+export class NpmApp {
+  readonly s3: S3 = new S3();
+  readonly s3Cache = new S3Cache(this.s3, BUCKET_NAME);
+
+  getNpmPackage(packageName: string) {
+    return new NpmPackage(this.s3Cache, NPM_CACHE_DOWNLOAD_URI, packageName);
+  }
+}
+
+// app singleton
+export const APP = new NpmApp();