diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000..6d6106f --- /dev/null +++ b/.babelrc @@ -0,0 +1,29 @@ +{ + "plugins": ["source-map-support", "@babel/plugin-transform-runtime"], + "presets": [ + ["@babel/preset-env", { + "targets": { + "node": "8.10" + } + }] + ], + "env": { + "production": { + "retainLines": false, + "minified": true + }, + "development": { + "retainLines": true, + "plugins": [ + "@babel/plugin-transform-runtime" + ] + }, + "test": { + "retainLines": true, + "plugins": [ + "@babel/plugin-transform-runtime", + "istanbul" + ] + } + } +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..857bbe1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,54 @@ +# Logs +logs +*.log +npm-debug.log* + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules +jspm_packages + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + +# vim +.*.sw* +Session.vim + +# Serverless +.webpack +.serverless + +# Jetbrains IDEs +.idea + +# misc +.DS_Store + +tests/*config.json +docker-compose.yml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ba284b9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false + +[*.{php,json}] +indent_style = space +indent_size = 4 diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..fc7a32e --- /dev/null +++ b/.eslintignore @@ -0,0 +1,2 @@ +**/node_modules/* +coverage/* diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..51a2e70 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,48 @@ +{ + root: true, + env: { + browser: false, + node: true, + es6: true + }, + extends: [ "eslint:recommended" ], + parserOptions: { + "sourceType": "module", + "ecmaVersion": 8, + "ecmaFeatures": { + "globalReturn": false, + "impliedStrict": false, + "jsx": false, + "allowImportExportEverywhere": true + } + }, + rules: { + // allow paren-less arrow functions + "arrow-parens": 0, + // allow async-await + "generator-star-spacing": 0, + // allow debugger during development + "no-debugger": 0, + "quotes": [2, "single"] + }, + overrides: [ + { + files: [ + "tests/*.test.js" + ], + env: { + jest: true // now **/*.test.js files' env has both es6 *and* jest + }, + // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813 + // "extends": ["plugin:jest/recommended"] + plugins: ["jest"], + rules: { + "jest/no-disabled-tests": "warn", + "jest/no-focused-tests": "error", + "jest/no-identical-title": "error", + "jest/prefer-to-have-length": "warn", + "jest/valid-expect": "error" + } + } + ] +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d0fb6d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,59 @@ +# Logs +logs +*.log +npm-debug.log* + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules +jspm_packages + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + +# vim +.*.sw* +Session.vim + +# Serverless +.webpack +.serverless + +# env +env.yml +.env + +# Jetbrains IDEs +.idea + +# misc +.DS_Store + +tests/*config.json +docker-compose.yml + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2224242 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM node:10 +LABEL maintainer="noogen " +ENV NPM_CONFIG_LOGLEVEL=warn \ + APP_VERSION=1.0.0 +EXPOSE 5000 + +RUN apt-get update && apt-get upgrade -y \ + && apt-get install git -y \ + && npm install -g pm2 \ + && mkdir -p /usr/local/gtincloud \ + && groupadd -r gtincloud && useradd -r -g gtincloud -d /usr/local/gtincloud gtincloud \ + && chown gtincloud:gtincloud /usr/local/gtincloud \ + && apt-get clean -y && apt-get autoclean -y \ + && apt-get autoremove --purge -y \ + && rm -rf /var/lib/apt/lists/* /var/lib/log/* /tmp/* /var/tmp/* + +USER gtincloud +RUN cd /usr/local/gtincloud \ + && git clone https://github.com/niiknow/gtin-cloud --branch ${APP_VERSION} /usr/local/gtincloud/app \ + && cd app && npm install +WORKDIR /usr/local/gtincloud/app + +CMD ["npm", "run", "prod"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..de6a6bd --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ + +MIT License + +Copyright (c) friends@niiknow.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b488cd3 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# GTIN Cloud +> How would a consumer get all of a GTIN Item Information by simply knowing the GTIN Number of the Item? And, without using a database? + +So, without using a database, we would need to create a convention to store the data in a folder structure that is easy to access and is high performance. + +# Storage Stragegy +Let say you have a ficticious GTIN number 00123456789012. Then the folder path for this GTIN number would be: 123/456/789/00123456789012/ + +1. Drop the first to 2 digits (00) +2. Split the next 9 numbers into 3 digits folder structure +3. And store the data in the folder that is the GTIN + +# The Power of 3s +- When storing in the cloud like AWS S3, the first 3 characters identify the partition AWS store the data. This improve the speed of access. +- Storing in 3 characters also prevent a folder from having too many file and folders; assuming that it will all be numeric, it will probably won't be greater than 1000 objects. Most cloud storage services, including AWS S3, also limit response/list to 1000 objects. This mean that, if we were to download/sync these files to a local storage, it will also help with local folder listing speed. + +# MIT diff --git a/docker-compose.example.yml b/docker-compose.example.yml new file mode 100644 index 0000000..89d730d --- /dev/null +++ b/docker-compose.example.yml @@ -0,0 +1,16 @@ +version: '3' +services: + gtincloud: + build: . + restart: always + ports: + - '5000:5000' + environment: + # NodeJS env + - PORT=5000 + # AWS access credentials + - AWS_ACCESS_KEY_ID=XXXXXXXXXX + - AWS_SECRET_ACCESS_KEY=XXXXXXXXXX + - AWS_DEFAULT_REGION=us-east-1 + # AWS S3 configuration + - AWS_BUCKET=XXXXXXXXXX diff --git a/env.yml.example b/env.yml.example new file mode 100644 index 0000000..9c9dd71 --- /dev/null +++ b/env.yml.example @@ -0,0 +1,13 @@ +# HOW TO USE: +# +# 1 Add environment variables for the various stages here +# 2 Rename this file to env.yml and uncomment it's usage +# in the serverless.yml. +# 3 Make sure to not commit this file. + +dev: + FORMBUCKET: gtin-cloud + DEBUG: gtin-cloud + +prod: + FORMBUCKET: gtin-cloud diff --git a/handler.js b/handler.js new file mode 100644 index 0000000..8665a0a --- /dev/null +++ b/handler.js @@ -0,0 +1,3 @@ +import store from './lib/storeHandler' + +export const storeHandler = post diff --git a/package.json b/package.json new file mode 100644 index 0000000..c1b3b66 --- /dev/null +++ b/package.json @@ -0,0 +1,62 @@ +{ + "name": "gtin-cloud", + "version": "1.3.0", + "description": "GTIN cloud storage strategy ", + "main": "handler.js", + "scripts": { + "deploy": "sls deploy && sls s3deploy", + "deploy-prod": "sls deploy --stage prod && sls s3deploy --stage prod", + "lint": "eslint --ext .js,.jsx ./", + "lint-fix": "eslint --fix --ext .js,.jsx ./", + "logs": "sls logs -f formPostHandler -t", + "serverless": "node_modules/.bin/serverless", + "sls": "node_modules/.bin/serverless", + "service-info": "sls info", + "local": "node_modules/.bin/serverless offline --stage dev", + "prod": "pm2 start pm2.json --no-daemon" + }, + "author": "Tom Noogen", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/niiknow/gtin-cloud.git" + }, + "dependencies": { + "busboy": "^0.2.14", + "consolidate": "^0.15.1", + "debug": "^4.1.1", + "mjml": "^4.3.1", + "nodemailer": "^4.7.0", + "nunjucks": "^3.2.0", + "recaptcha2": "^1.3.3", + "source-map-support": "^0.5.12", + "temp": "^0.8.3", + "uuid": "^3.3.2", + "zlib": "^1.0.5" + }, + "devDependencies": { + "@babel/cli": "^7.4.3", + "@babel/core": "^7.4.3", + "@babel/plugin-transform-runtime": "^7.4.3", + "@babel/preset-env": "^7.4.3", + "@babel/register": "^7.4.0", + "@babel/runtime": "^7.4.3", + "aws-sdk": "^2.437.0", + "babel-loader": "^8.0.5", + "babel-plugin-source-map-support": "^2.0.1", + "copy-webpack-plugin": "^4.6.0", + "cross-env": "^5.1.6", + "eslint": "^5.16.0", + "eslint-config-prettier": "^3.6.0", + "eslint-plugin-import": "^2.16.0", + "eslint-plugin-jest": "^21.27.2", + "jest": "^24.7.1", + "serverless": "^1.40.0", + "serverless-apigw-binary": "^0.4.4", + "serverless-offline": "^3.33.0", + "serverless-plugin-existing-s3": "^2.3.3", + "serverless-webpack": "^5.2.0", + "webpack": "^4.29.6", + "webpack-node-externals": "^1.7.2" + } +} diff --git a/pm2.json b/pm2.json new file mode 100644 index 0000000..954b8bb --- /dev/null +++ b/pm2.json @@ -0,0 +1,12 @@ +{ + "apps" : [{ + "name" : "gtin-cloud", + "script" : "app.js", + "instances" : "max", + "exec_mode" : "cluster", + "merge_logs" : true, + "log_date_format" : "YYYY-MM-DDTHH:mm:ss.SSSZ", + "listen_timeout" : 20000, + "kill_timeout" : 30000 + }] +} diff --git a/serverless.yml b/serverless.yml new file mode 100644 index 0000000..713bb4b --- /dev/null +++ b/serverless.yml @@ -0,0 +1,69 @@ +service: gtin-cloud + +# Use the serverless-webpack plugin to transpile ES6 +plugins: + - serverless-webpack + - serverless-plugin-existing-s3 + - serverless-apigw-binary + - serverless-offline + +# serverless-webpack configuration +# Enable auto-packing of external modules +custom: + webpack: + webpackConfig: ./webpack.config.js + includeModules: true + serverless-offline: + host: 0.0.0.0 + port: 5000 + corsAllowOrigin: "*" + corsAllowHeaders: "Origin, X-Requested-With, Content-Type, Accept" + apigwBinary: + types: + - 'application/x-www-form-urlencoded' + - 'multipart/form-data' + +provider: + name: aws + runtime: nodejs8.10 + stage: ${opt:stage, 'dev'} + region: us-east-1 + iamRoleStatements: + - Effect: "Allow" + Action: + - "s3:*" + Resource: + - "*" + # To load environment variables externally + # rename env.example to env.yml and uncomment + # the following line. Also, make sure to not + # commit your env.yml. + # + environment: ${file(env.yml):${self:provider.stage}} + +package: + exclude: + - demo/** + - node_modules/** + - tests/** + - .babelrc + - .d* + - .e* + - .g* + - .git/** + - .DS_* + - docker*.yml + - run-*.sh + +functions: + storeHandler: + handler: handler.storeHandler + events: + - http: + method: post + path: store/{gtin} + cors: true + request: + parameters: + paths: + id: true diff --git a/src/dateId.js b/src/dateId.js new file mode 100644 index 0000000..a5611eb --- /dev/null +++ b/src/dateId.js @@ -0,0 +1,12 @@ +/** + * Convert date to a number that enable most-recent first sort/ordering + * on storage system such as AWS S3 + * + * @param {Date} date the date object + * @return {string} the response + */ +export default (date) => { + const today = date.toISOString().slice(0, 16) + const todayNum = 999999999999 - parseInt(today.replace(/[-T:]+/gi, ''), 10) + return todayNum.toString() +} diff --git a/src/gtinPath.js b/src/gtinPath.js new file mode 100644 index 0000000..e2f8534 --- /dev/null +++ b/src/gtinPath.js @@ -0,0 +1,13 @@ +/** + * Parse gtin into a folder path + * + * Example: 00123456789012 becomes 123/456/789/00123456789012 + * + * @param {string} gtin the 14 digits Global Trade Item Number + * @return {string} the folder path + */ +export default (gtin) => { + const upc = gtin2.slice(-12); + const parts = [upc.substr(0, 3), upc.substr(3, 3), upc.substr(6, 3)]; + return parts.join('/') + '/' + gtin; +} diff --git a/src/response.js b/src/response.js new file mode 100644 index 0000000..d1bf3bd --- /dev/null +++ b/src/response.js @@ -0,0 +1,25 @@ +const rspHeaders = { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'OPTIONS, POST', + 'Access-Control-Allow-Headers': 'Content-Type' +} + +export default (rsp, callback) => { + return (data, code=200, headers=null) => { + const body = JSON.stringify({ status: code, message: data }) + if (callback) { + const rst = { + headers: headers || rspHeaders, + statusCode: code, + body: body + } + + return callback(null, rst) + } + + rsp.writeHead(code, headers || rspHeaders) + rsp.write(body) + rsp.end() + } +} diff --git a/src/storeHandler.js b/src/storeHandler.js new file mode 100644 index 0000000..092505f --- /dev/null +++ b/src/storeHandler.js @@ -0,0 +1,24 @@ +import fs from 'fs' +import res from './response' + +const debug = require('debug')('gtin-cloud') + +/** + * Handle form post of type: + * application/json + * application/x-www-form-urlencoded + * multipart/form-data + * + * @param object event the event + * @param object context the context + * @param Function callback the callback + */ +export default async (event, context, callback) => { + // Parameters + // @param string gtin the gtin + // @param string image_url the image url + // @param string data_url the data url + // @param string vendor the vendor (optional - default empty) + const gtin = event.pathParameters.gtin + +} diff --git a/webpack.config.js b/webpack.config.js new file mode 100644 index 0000000..f58eb97 --- /dev/null +++ b/webpack.config.js @@ -0,0 +1,36 @@ +const slsw = require('serverless-webpack') +const nodeExternals = require('webpack-node-externals') +const CopyWebpackPlugin = require('copy-webpack-plugin') + +module.exports = { + entry: slsw.lib.entries, + target: 'node', + // Generate sourcemaps for proper error messages + devtool: 'source-map', + // Since 'aws-sdk' is not compatible with webpack, + // we exclude all node dependencies + externals: [nodeExternals()], + mode: slsw.lib.webpack.isLocal ? 'development' : 'production', + optimization: { + // We do not want to minimize our code. + minimize: false, + }, + performance: { + // Turn off size warnings for entry points + hints: false, + }, + plugins: [ + new CopyWebpackPlugin(['forms/**', 'templates/**']) + ], + // Run babel on all .js files and skip those in node_modules + module: { + rules: [ + { + test: /\.js$/, + loader: 'babel-loader', + include: __dirname, + exclude: /node_modules/, + }, + ], + }, +}