feat: Feature/ingestion construct (#8)

* initial push of a the construct structure

.projenrc.ts

@@ -1,29 +1,44 @@
+/**
+ *  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ *  with the License. A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+ *  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
+ *  and limitations under the License.
+ */
 import { awscdk } from 'projen';
 
-const gitHubUser = 'aws-samples';
-const projectName = 'emerging-tech-cdk-constructs';
+// Constants
+const GITHUB_USER = 'aws-samples';
+const PROJECT_NAME = 'emerging-tech-cdk-constructs';
+const CDK_VERSION: string = '2.96.2';
+
 const project = new awscdk.AwsCdkConstructLibrary({
   author: 'Amazon Web Services - Prototyping and Cloud Engineering',
-  authorAddress: projectName+'@amazon.com',
-  cdkVersion: '2.96.2',
-  // Fill in the below during an upgrade
-  // cdkVersionPinning: true,
-  // constructsVersion: '10.2.70',
-  // projenVersion: '0.73.28',
-
+  authorAddress: PROJECT_NAME+'@amazon.com',
+  authorOrganization: true,
+  cdkVersion: CDK_VERSION,
+  projenVersion: '~0.73.33',
+  constructsVersion: '10.0.5',
+  cdkVersionPinning: true,
   defaultReleaseBranch: 'main',
   jsiiVersion: '~5.0.0',
-  name: projectName,
+  name: PROJECT_NAME,
   projenrcTs: true,
-  repositoryUrl: 'https://github.com/'+gitHubUser+'/'+projectName,
+  repositoryUrl: 'https://github.com/'+GITHUB_USER+'/'+PROJECT_NAME,
 
-  // deps: [],                /* Runtime dependencies of this module. */
   // description: undefined,  /* The description is just a string that helps people understand the purpose of the package. */
-  // devDeps: [],             /* Build dependencies for this module. */
-  packageName: '@'+gitHubUser+'/'+projectName, /* The "name" in package.json. */
+  packageName: '@'+GITHUB_USER+'/'+PROJECT_NAME, /* The "name" in package.json. */
+  keywords: ['constructs', 'aws-cdk', 'generative-ai', 'emerging-tech'],
+  devDeps: ['eslint-plugin-header'],
+  //peerDeps: [],
 
   // Keep synchronized with https://github.com/nodejs/release#release-schedule
-  minNodeVersion: '16.13.0', // 'MAINTENANCE' (first LTS)
+  minNodeVersion: '18.12.0', // 'MAINTENANCE' (first LTS)
   maxNodeVersion: '20.7.0', // 'CURRENT'
   workflowNodeVersion: '18.x', // 'ACTIVE'
 
@@ -33,12 +48,25 @@ const project = new awscdk.AwsCdkConstructLibrary({
   githubOptions: {
     pullRequestLintOptions: {
       contributorStatement: 'By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of the project license.',
+      contributorStatementOptions: {
+        exemptUsers: ['amazon-auto', 'dependabot[bot]', 'emerging-tech-cdk-constructs-bot', 'github-actions'],
+      },
     },
   },
   docgen: false,
   licensed: true,
   license: 'Apache-2.0',
+  copyrightPeriod: '2023-',
+  copyrightOwner: 'Amazon.com, Inc. or its affiliates. All Rights Reserved.',
   gitignore: ['*.DS_STORE'],
   stability: 'experimental',
+  sampleCode: false,
 });
+
+// Add License header automatically
+project.eslint?.addPlugins('header');
+project.eslint?.addRules({
+  'header/header': [2, 'header.js'],
+});
+
 project.synth();

DEVELOPER_GUIDE.md

@@ -45,10 +45,10 @@ npm install -g npm aws-cdk pnpm @aws/pdk projen
 | Action                                            | Explanation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | :-------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | (optional)<br/>`git checkout -b your-branch-name` | If you're working in a different branch than main in your forked repo and haven't changed your local branch, now is a good time to do so.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| `mkdir src/<construct name> `                     | Creates a dedicated folder to work on your construct.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| `cd src/<construct name>`                         | Change directory to the folder where you want to change code (this might also be `cd core`).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| *Do all your code editing*                        | Open your code editor and and create the construct or perform your edits on an existing construct (or core). The Common folder                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| `npx projen build`                                | This is the build command for the library. It will build, lint and run the unit and integration tests. If you make any substantive changes to the code, you will almost certainly see some or all of the tests fail. The next section will describe how testing works in AWS Emerging Tech Constructs and how to write, refresh and execute tests. In the meantime, you can at least check if your code transpiles correctly without running the tests by running `npm run build`. If you've edited any modules in /core in addition to a construct, be sure to build /core before building your construct. This command also packages locally the constructs to the /dist folder. |
+| `mkdir src/patterns/<emerging tech area>/<construct name> `                     | Creates a dedicated folder to work on your construct.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| `cd src/patterns/<emerging tech area>/<construct name>`                         | Change directory to the folder where you want to change code.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| *Do all your code editing*                        | Open your code editor and create the construct or perform your edits on an existing construct. Your construct code must be located in the src folder. Put only your Typescript files related to your construct in that folder. If you need to bundle additional code, add it to a separate folder in the root folder of this repo (see existing examples, like `lambda` and `resources` folders.) Use an existing construct as an example of the structure that is expected (architecture.png, README.md, index.ts). For the architecture diagram of your construct, please use the provided Draw.io project located in the /docs folder. Create a new tab with your construct name. Finally, export  your construct in the src/index.ts file. An example of the expected project structure is provided at the end of this document. Common code containing helper functions to standardize and accelerate development is located in the src/common folder.                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| `npx projen build`                                | This is the build command for the library. It will build, lint, add license header files and run the unit and integration tests. If you make any substantive changes to the code, you will almost certainly see some or all of the tests fail. The next section will describe how testing works in AWS Emerging Tech Constructs and how to write, refresh and execute tests. In the meantime, you can at least check if your code transpiles correctly without running the tests by running `npm run build`. If you've edited any modules in /core in addition to a construct, be sure to build /core before building your construct. This command also packages locally the constructs to the /dist folder. |
 
 ## Testing
 
@@ -66,4 +66,35 @@ All test files can be found in the /test directory under each construct (or core
 
 ---
 
+## Project structure
+
+```
+.
+|--docs/ (draw.io project containing architecture diagrams for all constructs)
+|--lib/ (Build output)
+|--lambda/ (Lambda functions code)
+|--resources (If you need additional resources packaged with your library)
+|--src/ (Source .ts files)
+    |--common/ (Common code reused accross constructs)
+        |--helpers
+            |-- README.md (Documentation for helper functions)
+            |-- *-helper.ts (Helper source file)
+    |--patterns/ (Constructs source files are here)
+        |--<emerging-tech-area>
+            |--<pattern-name>
+                |--index.ts (Construct source file)
+                |--README.md (Construct documentation)
+                |--architecture.png (Construct diagram)
+    |--index.ts (Constructs need to be exported from this index.ts file)
+|--test/
+    |--common/ (Common code reused accross constructs)
+        |--helpers
+            |-- *-helper.test.ts (Helper source file)
+    |--patterns/
+        |--<emerging-tech-area>
+            |--<pattern-name>
+                |--*.test.ts (construct test files)
+```
+
+
 &copy; Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

header.js

@@ -0,0 +1,12 @@
+/**
+ *  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ *  with the License. A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+ *  OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
+ *  and limitations under the License.
+ */

lambda/embeddings_job/src/Dockerfile

@@ -0,0 +1,18 @@
+FROM amazon/aws-lambda-python:latest
+
+# Installs python, removes cache file to make things smaller
+RUN yum update -y && \
+    yum install -y python3 python3-dev python3-pip gcc git && \
+    rm -Rf /var/cache/yum
+
+# Copies requirements.txt file into the container
+COPY requirements.txt ./
+# Installs dependencies found in your requirements.txt file
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+# Be sure to copy over the function itself!
+COPY . .
+
+# Points to the handler function of your lambda function
+CMD ["lambda.handler"]

lambda/embeddings_job/src/helpers/credentials_helper.py

@@ -0,0 +1,18 @@
+import json
+import boto3
+
+def get_credentials(secret_id: str, region_name: str) -> str:
+
+    client = boto3.client('secretsmanager', region_name=region_name)
+    response = client.get_secret_value(SecretId=secret_id)
+    secrets_value = json.loads(response['SecretString'])    
+
+    return secrets_value
+
+def get_credentials_string(secret_id: str, region_name: str) -> str:
+
+    client = boto3.client('secretsmanager', region_name=region_name)
+    response = client.get_secret_value(SecretId=secret_id)
+    secrets_value = response['SecretString']    
+
+    return secrets_value