From ab25a75ab6f622105ce7261027b116deba0e4b9a Mon Sep 17 00:00:00 2001 From: Type-Style Date: Wed, 7 Feb 2024 01:11:16 +0100 Subject: [PATCH 01/14] [Task] #18, limit request size for security reasons --- package-lock.json | 23 +++++++++++++++++++---- package.json | 3 ++- src/app.ts | 17 ++++++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4c0b355..e184a96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "express-validator": "^7.0.1", "helmet": "^7.1.0", "hpp": "^0.2.3", - "module-alias": "^2.2.3" + "module-alias": "^2.2.3", + "raw-body": "^2.5.2" }, "devDependencies": { "@jest/globals": "^29.7.0", @@ -2306,6 +2307,20 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/body-parser/node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -5518,9 +5533,9 @@ } }, "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", diff --git a/package.json b/package.json index 7e450e9..94d2f15 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "express-validator": "^7.0.1", "helmet": "^7.1.0", "hpp": "^0.2.3", - "module-alias": "^2.2.3" + "module-alias": "^2.2.3", + "raw-body": "^2.5.2" }, "_moduleAliases": { "@src": "dist" diff --git a/src/app.ts b/src/app.ts index 0f5b50a..3a743c2 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,6 +3,7 @@ import { config } from 'dotenv'; import express from 'express'; import helmet from 'helmet'; import hpp from 'hpp'; +import getRawBody from 'raw-body'; import cache from './cache'; import * as error from "./error"; import writeRouter from '@src/controller/write'; @@ -27,6 +28,20 @@ app.use( app.use(hpp()); app.use(cache); +app.use(function (req, res, next) { + if (!['POST', 'PUT', 'DELETE'].includes(req.method)) { + return next() + } + getRawBody(req, { + length: req.headers['content-length'], + limit: '1mb', + encoding: true + }, function (err) { + if (err) { return next(err) } + next() + }) +}) + // routes app.get('/', (req, res) => { res.send('Hello World, via TypeScript and Node.js!'); @@ -40,7 +55,7 @@ app.use('/read', readRouter); app.use('/', express.static(path.join(__dirname, 'httpdocs'), { extensions: ['html', 'txt', "pdf"], index: "start.html", -})) +})); // error handling app.use(error.notFound); From 5c61c66769aa0f6ad05e460192d5f2ea50a29212 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Wed, 7 Feb 2024 01:25:40 +0100 Subject: [PATCH 02/14] [Task] #43, introduce gzip to transfer data --- package-lock.json | 60 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 ++ src/app.ts | 3 ++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index e184a96..4829c14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "chalk": "^4.1.2", + "compression": "^1.7.4", "express": "^4.18.2", "express-validator": "^7.0.1", "helmet": "^7.1.0", @@ -20,6 +21,7 @@ "@jest/globals": "^29.7.0", "@tsconfig/node20": "^20.1.2", "@types/bcrypt": "^5.0.2", + "@types/compression": "^1.7.5", "@types/dotenv": "^8.2.0", "@types/express": "^4.17.21", "@types/hpp": "^0.2.5", @@ -1514,6 +1516,15 @@ "@types/node": "*" } }, + "node_modules/@types/compression": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.5.tgz", + "integrity": "sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==", + "dev": true, + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -2610,6 +2621,47 @@ "node": ">= 0.8" } }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", + "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "dependencies": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.16", + "debug": "2.6.9", + "on-headers": "~1.0.2", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/compression/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -5137,6 +5189,14 @@ "node": ">= 0.8" } }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/package.json b/package.json index 94d2f15..4c528b7 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@jest/globals": "^29.7.0", "@tsconfig/node20": "^20.1.2", "@types/bcrypt": "^5.0.2", + "@types/compression": "^1.7.5", "@types/dotenv": "^8.2.0", "@types/express": "^4.17.21", "@types/hpp": "^0.2.5", @@ -37,6 +38,7 @@ }, "dependencies": { "chalk": "^4.1.2", + "compression": "^1.7.4", "express": "^4.18.2", "express-validator": "^7.0.1", "helmet": "^7.1.0", diff --git a/src/app.ts b/src/app.ts index 3a743c2..99eee4d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,6 +1,7 @@ require('module-alias/register'); import { config } from 'dotenv'; import express from 'express'; +import compression from 'compression'; import helmet from 'helmet'; import hpp from 'hpp'; import getRawBody from 'raw-body'; @@ -24,7 +25,7 @@ app.use( } }) ); - +app.use(compression()) app.use(hpp()); app.use(cache); From 05a726c49430b3397867ad5f2cb43c068782acbe Mon Sep 17 00:00:00 2001 From: Type-Style Date: Wed, 7 Feb 2024 14:17:15 +0100 Subject: [PATCH 03/14] [Task] #34 improve error handling, log server shutdowns --- src/app.ts | 38 +++++++++++++++++++++++++++++++------- src/error.ts | 2 +- src/scripts/logger.ts | 21 +++++++++++++++------ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/app.ts b/src/app.ts index 99eee4d..19a6670 100644 --- a/src/app.ts +++ b/src/app.ts @@ -9,7 +9,7 @@ import cache from './cache'; import * as error from "./error"; import writeRouter from '@src/controller/write'; import readRouter from '@src/controller/read'; -import path from 'path'; +import path from 'path'; import logger from '@src/scripts/logger'; // configurations @@ -19,7 +19,7 @@ app.use( helmet({ contentSecurityPolicy: { directives: { - "default-src": "'self'", + "default-src": "'self'", "img-src": "*" } } @@ -45,7 +45,12 @@ app.use(function (req, res, next) { // routes app.get('/', (req, res) => { - res.send('Hello World, via TypeScript and Node.js!'); + res.send('Hello World, via TypeScript and Node.js!'); +}); + +app.get('/test', (req, res) => { + res.send('Hello Test!'); + process.exit(); }); @@ -55,7 +60,7 @@ app.use('/read', readRouter); // use httpdocs as static folder app.use('/', express.static(path.join(__dirname, 'httpdocs'), { extensions: ['html', 'txt', "pdf"], - index: "start.html", + index: ["start.html", "start.txt"] , })); // error handling @@ -63,12 +68,31 @@ app.use(error.notFound); app.use(error.handler); // init server -app.listen(80, () => { - logger.log(`Server running //localhost:80, ENV: ${process.env.NODE_ENV}`, true); +const server = app.listen(80, () => { + logger.log(`Server running //localhost:80, ENV: ${process.env.NODE_ENV}`, true); +}); + +// catching shutdowns +['SIGINT', 'SIGTERM', 'exit'].forEach((signal) => { + process.on(signal, () => { + function logAndExit() { + // calling .shutdown allows your process to exit normally + // toobusy.shutdown(); + logger.log(`Server shutdown on signal: ${signal} //localhost:80`, true); + process.exit(); + } + if (signal != "exit") { // give the server time to shutdown before closing + server.close(logAndExit); + } else { + logger.log(`Server shutdown immediate: ${signal} //localhost:80`, true); + } + }); }); -process.on('uncaughtException', function(err) { +// last resort error handling +process.on('uncaughtException', function (err) { console.error('Caught exception:', err); logger.error(err); + server.close(); process.exit(1); }); \ No newline at end of file diff --git a/src/error.ts b/src/error.ts index 7b8d624..1b6ea58 100644 --- a/src/error.ts +++ b/src/error.ts @@ -36,7 +36,7 @@ export function handler(err: Error, req: Request, res: Response stack: process.env.NODE_ENV === "development" ? err.stack : "---" }; - logger.error(responseBody); + logger.error(JSON.stringify(responseBody)); res.json(responseBody); next(); } diff --git a/src/scripts/logger.ts b/src/scripts/logger.ts index 0beeaea..07b43ac 100644 --- a/src/scripts/logger.ts +++ b/src/scripts/logger.ts @@ -1,13 +1,22 @@ // primitive text logger -import fs from 'fs'; // typescript will compile to require -import path from 'path'; // typescript will compile to require -import chalk from "chalk"; // keep import syntax after compile +import fs from 'fs'; +import path from 'path'; +import chalk from "chalk"; -const logPath = path.resolve(__dirname, '../httpdocs', 'log.txt'); + +const dirPath = path.resolve(__dirname, '../httpdocs/log'); +const logPath = path.resolve(dirPath, 'start.txt'); + +if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + console.log("path created") +} + +// const logPath = path.resolve(__dirname, '../httpdocs/log', 'start.txt'); const date = new Date().toLocaleString('de-DE', { hour12: false }); export default { - log: (message:string|JSON, showDateInConsole:boolean=false, showLogInTest=false) => { + log: (message: string | JSON, showDateInConsole: boolean = false, showLogInTest = false) => { message = JSON.stringify(message); fs.appendFileSync(logPath, `${date} \t|\t ${message} \n`); if (showDateInConsole) { @@ -17,7 +26,7 @@ export default { console.log(message); } }, - error: (message:string|JSON|Response.Error) => { + error: (message: string | JSON | Response.Error) => { fs.appendFileSync(logPath, `${date} \t|\t ERROR: ${message} \n`); console.error(message); } From bdab23adbee95ca28589a7579993333bd03c3c53 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Wed, 7 Feb 2024 14:38:01 +0100 Subject: [PATCH 04/14] [Task] #34 installed and integrated tooBusy to send 503 when load is high --- package-lock.json | 18 +++++++++++++++++- package.json | 4 +++- src/app.ts | 40 ++++++++++++++++++++++++---------------- src/scripts/logger.ts | 1 - 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4829c14..d4c3d8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,8 @@ "helmet": "^7.1.0", "hpp": "^0.2.3", "module-alias": "^2.2.3", - "raw-body": "^2.5.2" + "raw-body": "^2.5.2", + "toobusy-js": "^0.5.1" }, "devDependencies": { "@jest/globals": "^29.7.0", @@ -27,6 +28,7 @@ "@types/hpp": "^0.2.5", "@types/jest": "^29.5.11", "@types/node": "^20.10.6", + "@types/toobusy-js": "^0.5.4", "@typescript-eslint/eslint-plugin": "^6.18.1", "@typescript-eslint/parser": "^6.18.1", "axios": "^1.6.5", @@ -1692,6 +1694,12 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, + "node_modules/@types/toobusy-js": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/@types/toobusy-js/-/toobusy-js-0.5.4.tgz", + "integrity": "sha512-hsKMbYiaL3ZWx7B3FYyN0rEJexw7I1HgKbNToX3ZZJv6373to954wlA7zrXR3/XoVwZnFwWqFguBs91sNzJGKQ==", + "dev": true + }, "node_modules/@types/yargs": { "version": "17.0.32", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.32.tgz", @@ -6106,6 +6114,14 @@ "node": ">=0.6" } }, + "node_modules/toobusy-js": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/toobusy-js/-/toobusy-js-0.5.1.tgz", + "integrity": "sha512-GiCux/c8G2TV0FTDgtxnXOxmSAndaI/9b1YxT14CqyeBDtTZAcJLx9KlXT3qECi8D0XCc78T4sN/7gWtjRyCaA==", + "engines": { + "node": ">=0.9.1" + } + }, "node_modules/touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", diff --git a/package.json b/package.json index 4c528b7..f78fb59 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@types/hpp": "^0.2.5", "@types/jest": "^29.5.11", "@types/node": "^20.10.6", + "@types/toobusy-js": "^0.5.4", "@typescript-eslint/eslint-plugin": "^6.18.1", "@typescript-eslint/parser": "^6.18.1", "axios": "^1.6.5", @@ -44,7 +45,8 @@ "helmet": "^7.1.0", "hpp": "^0.2.3", "module-alias": "^2.2.3", - "raw-body": "^2.5.2" + "raw-body": "^2.5.2", + "toobusy-js": "^0.5.1" }, "_moduleAliases": { "@src": "dist" diff --git a/src/app.ts b/src/app.ts index 19a6670..8074341 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,6 +1,7 @@ require('module-alias/register'); import { config } from 'dotenv'; import express from 'express'; +import toobusy from 'toobusy-js'; import compression from 'compression'; import helmet from 'helmet'; import hpp from 'hpp'; @@ -13,21 +14,26 @@ import path from 'path'; import logger from '@src/scripts/logger'; // configurations -config(); +config(); // dotenv + const app = express(); -app.use( - helmet({ - contentSecurityPolicy: { - directives: { - "default-src": "'self'", - "img-src": "*" - } +app.use(helmet({ + contentSecurityPolicy: { + directives: { + "default-src": "'self'", + "img-src": "*" } - }) -); + } +})); +app.use((req, res, next) => { + if (toobusy()) { + res.status(503).send("I'm busy right now, sorry."); + // todo add headers retry after and no cache + } else { next(); } +}); +app.use(cache); app.use(compression()) app.use(hpp()); -app.use(cache); app.use(function (req, res, next) { if (!['POST', 'PUT', 'DELETE'].includes(req.method)) { @@ -48,9 +54,11 @@ app.get('/', (req, res) => { res.send('Hello World, via TypeScript and Node.js!'); }); -app.get('/test', (req, res) => { - res.send('Hello Test!'); - process.exit(); +app.get('/test', function (req, res) { + // processing the request requires some work! + let i = 0; + while (i < 1e10) i++; + res.send("I counted to " + i); }); @@ -60,7 +68,7 @@ app.use('/read', readRouter); // use httpdocs as static folder app.use('/', express.static(path.join(__dirname, 'httpdocs'), { extensions: ['html', 'txt', "pdf"], - index: ["start.html", "start.txt"] , + index: ["start.html", "start.txt"], })); // error handling @@ -77,7 +85,7 @@ const server = app.listen(80, () => { process.on(signal, () => { function logAndExit() { // calling .shutdown allows your process to exit normally - // toobusy.shutdown(); + toobusy.shutdown(); logger.log(`Server shutdown on signal: ${signal} //localhost:80`, true); process.exit(); } diff --git a/src/scripts/logger.ts b/src/scripts/logger.ts index 07b43ac..8de1742 100644 --- a/src/scripts/logger.ts +++ b/src/scripts/logger.ts @@ -9,7 +9,6 @@ const logPath = path.resolve(dirPath, 'start.txt'); if (!fs.existsSync(dirPath)) { fs.mkdirSync(dirPath, { recursive: true }); - console.log("path created") } // const logPath = path.resolve(__dirname, '../httpdocs/log', 'start.txt'); From 8cf66b424b29587228e1c9bd132f268d0a8893e2 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Wed, 7 Feb 2024 14:45:18 +0100 Subject: [PATCH 05/14] [Task] #34 improved tooBusy, improved formatting --- src/app.ts | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/src/app.ts b/src/app.ts index 8074341..9502854 100644 --- a/src/app.ts +++ b/src/app.ts @@ -25,25 +25,18 @@ app.use(helmet({ } } })); -app.use((req, res, next) => { - if (toobusy()) { - res.status(503).send("I'm busy right now, sorry."); - // todo add headers retry after and no cache - } else { next(); } +app.use((req, res, next) => { // monitor eventloop to block requests if busy + if (toobusy()) { res.status(503).set({ 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Retry-After': '60' }).send("I'm busy right now, sorry."); } + else { next(); } }); app.use(cache); app.use(compression()) app.use(hpp()); -app.use(function (req, res, next) { - if (!['POST', 'PUT', 'DELETE'].includes(req.method)) { - return next() - } - getRawBody(req, { - length: req.headers['content-length'], - limit: '1mb', - encoding: true - }, function (err) { +app.use(function (req, res, next) { // limit request size limit when recieving data + if (!['POST', 'PUT', 'DELETE'].includes(req.method)) { return next(); } + getRawBody(req, { length: req.headers['content-length'], limit: '1mb', encoding: true }, + function (err) { if (err) { return next(err) } next() }) @@ -54,14 +47,6 @@ app.get('/', (req, res) => { res.send('Hello World, via TypeScript and Node.js!'); }); -app.get('/test', function (req, res) { - // processing the request requires some work! - let i = 0; - while (i < 1e10) i++; - res.send("I counted to " + i); -}); - - app.use('/write', writeRouter); app.use('/read', readRouter); From ffcca9a6dcffa0c44d4862ab7c443d50590cd0d2 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Thu, 8 Feb 2024 17:45:08 +0100 Subject: [PATCH 06/14] [Task, Temp] #41 installed ratelimiter and slowDown --- package-lock.json | 30 ++++++++++++++++++++++++++++++ package.json | 2 ++ src/app.ts | 38 ++++++++++++++++++++++++-------------- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index d4c3d8e..085305f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,8 @@ "chalk": "^4.1.2", "compression": "^1.7.4", "express": "^4.18.2", + "express-rate-limit": "^7.1.5", + "express-slow-down": "^2.0.1", "express-validator": "^7.0.1", "helmet": "^7.1.0", "hpp": "^0.2.3", @@ -3423,6 +3425,34 @@ "node": ">= 0.10.0" } }, + "node_modules/express-rate-limit": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.1.5.tgz", + "integrity": "sha512-/iVogxu7ueadrepw1bS0X0kaRC/U0afwiYRSLg68Ts+p4Dc85Q5QKsOnPS/QUjPMHvOJQtBDrZgvkOzf8ejUYw==", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": "4 || 5 || ^5.0.0-beta.1" + } + }, + "node_modules/express-slow-down": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/express-slow-down/-/express-slow-down-2.0.1.tgz", + "integrity": "sha512-zRogSZhNXJYKDBekhgFfFXGrOngH7Fub7Mx2g8OQ4RUBwSJP/3TVEKMgSGR/WlneT0mJ6NBUnidHhIELGVPe3w==", + "dependencies": { + "express-rate-limit": "7" + }, + "engines": { + "node": ">= 16" + }, + "peerDependencies": { + "express": ">= 4" + } + }, "node_modules/express-validator": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/express-validator/-/express-validator-7.0.1.tgz", diff --git a/package.json b/package.json index f78fb59..8b59750 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,8 @@ "chalk": "^4.1.2", "compression": "^1.7.4", "express": "^4.18.2", + "express-rate-limit": "^7.1.5", + "express-slow-down": "^2.0.1", "express-validator": "^7.0.1", "helmet": "^7.1.0", "hpp": "^0.2.3", diff --git a/src/app.ts b/src/app.ts index 9502854..6fa9452 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,6 +2,8 @@ require('module-alias/register'); import { config } from 'dotenv'; import express from 'express'; import toobusy from 'toobusy-js'; +// import { rateLimit } from 'express-rate-limit'; +// import { slowDown } from 'express-slow-down'; import compression from 'compression'; import helmet from 'helmet'; import hpp from 'hpp'; @@ -17,29 +19,37 @@ import logger from '@src/scripts/logger'; config(); // dotenv const app = express(); -app.use(helmet({ - contentSecurityPolicy: { - directives: { - "default-src": "'self'", - "img-src": "*" - } - } -})); + app.use((req, res, next) => { // monitor eventloop to block requests if busy if (toobusy()) { res.status(503).set({ 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Retry-After': '60' }).send("I'm busy right now, sorry."); } else { next(); } }); + +// const slowDownLimiter = slowDown({ +// windowMs: 1 * 60 * 1000, +// delayAfter: 5, // Allow 5 requests per 15 minutes. +// delayMs: (used) => (used - 5) * 1000, // Add delay after delayAfter is reached +// }) + +// const rateLimiter = rateLimit({ +// windowMs: 1 * 60 * 1000, +// max: 10, // Limit each IP per `window` +// standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers +// legacyHeaders: false, // Disable the `X-RateLimit-*` headers +// }) + +app.use(helmet({ contentSecurityPolicy: { directives: { "default-src": "'self'", "img-src": "*" } } })); app.use(cache); app.use(compression()) app.use(hpp()); - app.use(function (req, res, next) { // limit request size limit when recieving data if (!['POST', 'PUT', 'DELETE'].includes(req.method)) { return next(); } - getRawBody(req, { length: req.headers['content-length'], limit: '1mb', encoding: true }, - function (err) { - if (err) { return next(err) } - next() - }) + getRawBody(req, { length: req.headers['content-length'], limit: '1mb', encoding: true }, + function (err) { + if (err) { return next(err) } + next() + } + ) }) // routes From 23960260c868824b491c9a9cb71338efa2f545ba Mon Sep 17 00:00:00 2001 From: Type-Style Date: Fri, 9 Feb 2024 10:40:49 +0100 Subject: [PATCH 07/14] [Task] #42 cleanup ipv6 addresses --- src/app.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/app.ts b/src/app.ts index 6fa9452..e9d1228 100644 --- a/src/app.ts +++ b/src/app.ts @@ -21,9 +21,14 @@ config(); // dotenv const app = express(); app.use((req, res, next) => { // monitor eventloop to block requests if busy - if (toobusy()) { res.status(503).set({ 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Retry-After': '60' }).send("I'm busy right now, sorry."); } - else { next(); } + if (toobusy()) { + res.status(503).set({ 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Retry-After': '60' }).send("I'm busy right now, sorry."); + } else { next(); } }); +app.use((req, res, next) => { // clean up IPv6 Addresses + if (req.ip) { res.locals.ip = req.ip.startsWith('::ffff:') ? req.ip.substring(7) : req.ip; } + next(); +}) // const slowDownLimiter = slowDown({ // windowMs: 1 * 60 * 1000, @@ -33,7 +38,7 @@ app.use((req, res, next) => { // monitor eventloop to block requests if busy // const rateLimiter = rateLimit({ // windowMs: 1 * 60 * 1000, -// max: 10, // Limit each IP per `window` +// limit: 10, // Limit each IP per `window` // standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers // legacyHeaders: false, // Disable the `X-RateLimit-*` headers // }) @@ -54,7 +59,7 @@ app.use(function (req, res, next) { // limit request size limit when recieving d // routes app.get('/', (req, res) => { - res.send('Hello World, via TypeScript and Node.js!'); + res.send('Hello World, via TypeScript and Node.js! ' + res.locals.ip); }); app.use('/write', writeRouter); From 3a066a5c018033efb20c4786e8caf15fa44b43ff Mon Sep 17 00:00:00 2001 From: Type-Style Date: Fri, 9 Feb 2024 16:12:22 +0100 Subject: [PATCH 08/14] [Change] #10 error handling for better gitBash and txt output, also reduced stack in case of validation errors --- src/error.ts | 4 ++-- src/scripts/logger.ts | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/error.ts b/src/error.ts index 1b6ea58..2eaf6e8 100644 --- a/src/error.ts +++ b/src/error.ts @@ -29,14 +29,14 @@ export function handler(err: Error, req: Request, res: Response message = err.message; } - const responseBody = { + const responseBody:Response.Error = { status: statusCode, name: err.name, message: message, stack: process.env.NODE_ENV === "development" ? err.stack : "---" }; - logger.error(JSON.stringify(responseBody)); + logger.error(responseBody); res.json(responseBody); next(); } diff --git a/src/scripts/logger.ts b/src/scripts/logger.ts index 8de1742..e5c9609 100644 --- a/src/scripts/logger.ts +++ b/src/scripts/logger.ts @@ -3,7 +3,6 @@ import fs from 'fs'; import path from 'path'; import chalk from "chalk"; - const dirPath = path.resolve(__dirname, '../httpdocs/log'); const logPath = path.resolve(dirPath, 'start.txt'); @@ -25,8 +24,19 @@ export default { console.log(message); } }, - error: (message: string | JSON | Response.Error) => { - fs.appendFileSync(logPath, `${date} \t|\t ERROR: ${message} \n`); - console.error(message); + error: (content: string | Response.Error) => { + fs.appendFileSync(logPath, `${date} \t|\t [ERROR]: ${JSON.stringify(content)} \n`); + if (process.env.NODE_ENV == "production") { return; } + if (typeof content != "string" && Object.hasOwnProperty.call(content, "message")) { + const messageAsString = JSON.stringify(content.message); + if (content.stack) { // replace redundant information + content.stack = content.stack.replace(messageAsString,""); + } + const consoleMessage = structuredClone(content); // create clone so response output is not "further" affected + consoleMessage.message = messageAsString; // gitbash output improvement (w/o objects in arrays appear as [Object]) + content = consoleMessage; + } + console.error(content); // log string right away or processed Object + } } From 22f28960c62a8dcc46008dc2fe52bb8988469773 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Sat, 10 Feb 2024 01:26:36 +0100 Subject: [PATCH 09/14] [Task] #41 prepare Log for RateLImit errors --- src/scripts/logger.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/scripts/logger.ts b/src/scripts/logger.ts index e5c9609..8978fef 100644 --- a/src/scripts/logger.ts +++ b/src/scripts/logger.ts @@ -25,18 +25,23 @@ export default { } }, error: (content: string | Response.Error) => { - fs.appendFileSync(logPath, `${date} \t|\t [ERROR]: ${JSON.stringify(content)} \n`); + // logfile + const applyErrorPrefix = !/^\[\w+\]/.test(typeof content == "string" ? content : content.message); + const logMessageTemplate = `${date} \t|\t${applyErrorPrefix ? ' [ERROR]' : ''} ${typeof content == "string" ? content : JSON.stringify(content.message) } \n`; + fs.appendFileSync(logPath, logMessageTemplate); if (process.env.NODE_ENV == "production") { return; } + + // console if (typeof content != "string" && Object.hasOwnProperty.call(content, "message")) { const messageAsString = JSON.stringify(content.message); if (content.stack) { // replace redundant information - content.stack = content.stack.replace(messageAsString,""); + content.stack = content.stack.replace(messageAsString, ""); } const consoleMessage = structuredClone(content); // create clone so response output is not "further" affected consoleMessage.message = messageAsString; // gitbash output improvement (w/o objects in arrays appear as [Object]) content = consoleMessage; } console.error(content); // log string right away or processed Object - + } } From c79dee8b575ca24bad0bcc7d9b0b3fa6f9cdbed0 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Sat, 10 Feb 2024 01:29:13 +0100 Subject: [PATCH 10/14] [Temp] #41 write route rateLImited temp: see Todos --- src/app.ts | 14 +++++- src/controller/write.ts | 95 ++++++++++++++++++++++++++++++++++------- types.d.ts | 8 ++++ 3 files changed, 100 insertions(+), 17 deletions(-) diff --git a/src/app.ts b/src/app.ts index e9d1228..11f75e1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -15,6 +15,9 @@ import readRouter from '@src/controller/read'; import path from 'path'; import logger from '@src/scripts/logger'; +// console.log({ "status": 403, "name": "Error", "message": { "errors": [{ "type": "field", "msg": "Invalid value", "path": "user", "location": "query" }, { "type": "field", "msg": "is required", "path": "lat", "location": "query" }]}}); +// console.log(JSON.stringify({ "status": 403, "name": "Error", "message": { "errors": [{ "type": "field", "msg": "Invalid value", "path": "user", "location": "query" }, { "type": "field", "msg": "is required", "path": "lat", "location": "query" }]}}, null, 2)); + // configurations config(); // dotenv @@ -26,8 +29,15 @@ app.use((req, res, next) => { // monitor eventloop to block requests if busy } else { next(); } }); app.use((req, res, next) => { // clean up IPv6 Addresses - if (req.ip) { res.locals.ip = req.ip.startsWith('::ffff:') ? req.ip.substring(7) : req.ip; } - next(); + if (req.ip) { + res.locals.ip = req.ip.startsWith('::ffff:') ? req.ip.substring(7) : req.ip; + next(); + } else { + const message = "No IP provided" + logger.error(message); + res.status(400).send(message); + } + }) // const slowDownLimiter = slowDown({ diff --git a/src/controller/write.ts b/src/controller/write.ts index dc0658d..47f7d6b 100644 --- a/src/controller/write.ts +++ b/src/controller/write.ts @@ -1,19 +1,83 @@ import express, { Request, Response, NextFunction } from 'express'; import { entry } from '@src/models/entry'; import { validationResult } from 'express-validator'; -import { create as createError } from '@src/error'; +import { create as createError } from '@src/error'; +import { slowDown, Options as slowDownOptions } from 'express-slow-down'; +import { rateLimit, Options as rateLimiterOptions } from 'express-rate-limit'; +import logger from '@src/scripts/logger'; + +// TODO clean up after 1 day? +// TODO move rateLimit to own file +const ipsThatReachedLimit: RateLimit.obj = {}; + +const baseOptions: Partial = { + windowMs: 5 * 60 * 1000, + //skip: (req, res) => (res.locals.ip == process.env.LOCALHOST) +} + +const baseSlowDown: Partial = { + ...baseOptions, + delayAfter: 3, // no delay for amount of attempts + delayMs: (used: number) => (used - 3) * 125, // Add delay after delayAfter is reached +} + +const baseRateLimit: Partial = { + ...baseOptions, + limit: 10, // Limit each IP per window + standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers + legacyHeaders: false, // Disable the `X-RateLimit-*` headers +} + +const errorRateLimiter = rateLimit({ + ...baseRateLimit, + message: 'Too many requests with errors', + handler: (req: Request, res: Response, next: NextFunction, options: rateLimiterOptions) => { + if (!Object.prototype.hasOwnProperty.call(ipsThatReachedLimit, res.locals.ip)) { + logger.error(`[RateLimit] for invalid requests reached ${res.locals.ip}, ${req.get('User-Agent')}`); + ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: true }; + } + res.status(options.statusCode).send(options.message); + } +}); + +// function customRateLimit(req: Request, res: Response, next: NextFunction) { +// console.count("customRateLimit"); +// if (!validationResult(req).isEmpty()) { + +// } else { +// rateLimit({ +// ...baseRateLimit, +// limit: 20, +// handler: (req: Request, res: Response, next: NextFunction, options: rateLimiterOptions) => { +// if (!Object.prototype.hasOwnProperty.call(ipsThatReachedLimit, res.locals.ip)) { +// logger.error(`[RateLimit] for valid requests reached ${res.locals.ip}, ${req.get('User-Agent')}`); +// ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: false }; +// } +// res.status(options.statusCode).send(options.message); +// } +// }) +// } +// } + // example call: /write?user=xx&lat=00.000&lon=00.000×tamp=1704063600000&hdop=0.0&altitude=0.000&speed=0.000&heading=000.0 -async function errorChecking (req:Request, res:Response, next:NextFunction) { + +function errorChecking(req: Request, res: Response, next: NextFunction) { const errors = validationResult(req); if (!errors.isEmpty()) { - const errorAsJson = { errors: errors.array()}; - const errorAsString = JSON.stringify(errorAsJson); - const hasKeyErrors = errors.array().some(error => error.msg.includes("Key")); - - // send forbidden or unprocessable content - return createError(res, hasKeyErrors ? 403 : 422, errorAsString, next) + + // if errors happend, then rateLimit to prevent key bruteforcing + errorRateLimiter(req, res, () => { + const errorAsJson = { errors: errors.array() }; + const errorAsString = JSON.stringify(errorAsJson); + const hasKeyErrors = errors.array().some(error => error.msg.includes("Key")); + + // send forbidden or unprocessable content + return createError(res, hasKeyErrors ? 403 : 422, errorAsString, next) + }); + + return; } if (req.method == "HEAD") { @@ -21,22 +85,23 @@ async function errorChecking (req:Request, res:Response, next:NextFunction) { return; } + next(); +} + +async function writeData(req: Request, res: Response, next: NextFunction) { // Regular Save logic from here await entry.create(req, res, next); - if (!res.locals.error) { + if (!res.locals.error) { res.send(req.query); - } else { - /* at this point error handling already happend, - * or the request has already been send - * therefor there is no need for it again (only middleware to follow at this point) */ + } else { next(); } } const router = express.Router(); -router.get('/', entry.validate, errorChecking); -router.head('/', entry.validate, errorChecking); +router.get('/', slowDown(baseSlowDown), entry.validate, errorChecking, writeData); +router.head('/', slowDown(baseSlowDown), entry.validate, errorChecking); export default router; \ No newline at end of file diff --git a/types.d.ts b/types.d.ts index 7cb4c68..064db1f 100644 --- a/types.d.ts +++ b/types.d.ts @@ -1,6 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ +namespace RateLimit { + interface obj { + [key: string]: { + limitReachedOnError: boolean + } + } +} + namespace Response { interface Message { message: string; From aad9b347382534e7bd990623542b16e189aaa6ab Mon Sep 17 00:00:00 2001 From: Type-Style Date: Tue, 13 Feb 2024 09:25:09 +0100 Subject: [PATCH 11/14] [Task] #34 colorize prefix in console --- src/scripts/logger.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/scripts/logger.ts b/src/scripts/logger.ts index 8978fef..67effdd 100644 --- a/src/scripts/logger.ts +++ b/src/scripts/logger.ts @@ -40,6 +40,11 @@ export default { const consoleMessage = structuredClone(content); // create clone so response output is not "further" affected consoleMessage.message = messageAsString; // gitbash output improvement (w/o objects in arrays appear as [Object]) content = consoleMessage; + } else if (typeof content == "string") { + const prefix = content.match(/^\[\w+\]/); + if (prefix?.length) { + content = content.replace(prefix[0], chalk.red(prefix[0])); + } } console.error(content); // log string right away or processed Object From db7558ed8e3755cb752632a97983bc250a696bf2 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Tue, 13 Feb 2024 09:25:51 +0100 Subject: [PATCH 12/14] [Task] #42 extract middlewares and move to folder --- src/app.ts | 4 +-- src/controller/read.ts | 2 +- src/controller/write.ts | 66 +++-------------------------------- src/{ => middleware}/cache.ts | 0 src/{ => middleware}/error.ts | 0 src/middleware/limit.ts | 44 +++++++++++++++++++++++ src/models/entry.ts | 2 +- src/scripts/file.ts | 2 +- 8 files changed, 53 insertions(+), 67 deletions(-) rename src/{ => middleware}/cache.ts (100%) rename src/{ => middleware}/error.ts (100%) create mode 100644 src/middleware/limit.ts diff --git a/src/app.ts b/src/app.ts index 11f75e1..3761d0b 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,8 +8,8 @@ import compression from 'compression'; import helmet from 'helmet'; import hpp from 'hpp'; import getRawBody from 'raw-body'; -import cache from './cache'; -import * as error from "./error"; +import cache from './middleware/cache'; +import * as error from "./middleware/error"; import writeRouter from '@src/controller/write'; import readRouter from '@src/controller/read'; import path from 'path'; diff --git a/src/controller/read.ts b/src/controller/read.ts index f17c93c..02294dc 100644 --- a/src/controller/read.ts +++ b/src/controller/read.ts @@ -1,6 +1,6 @@ import express, { Request, Response, NextFunction } from 'express'; import * as file from '@src/scripts/file'; -import { create as createError } from '@src/error'; +import { create as createError } from '@src/middleware/error'; import { validationResult, query } from 'express-validator'; const router = express.Router(); diff --git a/src/controller/write.ts b/src/controller/write.ts index 47f7d6b..7b405ff 100644 --- a/src/controller/write.ts +++ b/src/controller/write.ts @@ -1,72 +1,14 @@ import express, { Request, Response, NextFunction } from 'express'; import { entry } from '@src/models/entry'; import { validationResult } from 'express-validator'; -import { create as createError } from '@src/error'; -import { slowDown, Options as slowDownOptions } from 'express-slow-down'; -import { rateLimit, Options as rateLimiterOptions } from 'express-rate-limit'; -import logger from '@src/scripts/logger'; - -// TODO clean up after 1 day? -// TODO move rateLimit to own file -const ipsThatReachedLimit: RateLimit.obj = {}; - -const baseOptions: Partial = { - windowMs: 5 * 60 * 1000, - //skip: (req, res) => (res.locals.ip == process.env.LOCALHOST) -} - -const baseSlowDown: Partial = { - ...baseOptions, - delayAfter: 3, // no delay for amount of attempts - delayMs: (used: number) => (used - 3) * 125, // Add delay after delayAfter is reached -} - -const baseRateLimit: Partial = { - ...baseOptions, - limit: 10, // Limit each IP per window - standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers - legacyHeaders: false, // Disable the `X-RateLimit-*` headers -} - -const errorRateLimiter = rateLimit({ - ...baseRateLimit, - message: 'Too many requests with errors', - handler: (req: Request, res: Response, next: NextFunction, options: rateLimiterOptions) => { - if (!Object.prototype.hasOwnProperty.call(ipsThatReachedLimit, res.locals.ip)) { - logger.error(`[RateLimit] for invalid requests reached ${res.locals.ip}, ${req.get('User-Agent')}`); - ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: true }; - } - res.status(options.statusCode).send(options.message); - } -}); - -// function customRateLimit(req: Request, res: Response, next: NextFunction) { -// console.count("customRateLimit"); -// if (!validationResult(req).isEmpty()) { - -// } else { -// rateLimit({ -// ...baseRateLimit, -// limit: 20, -// handler: (req: Request, res: Response, next: NextFunction, options: rateLimiterOptions) => { -// if (!Object.prototype.hasOwnProperty.call(ipsThatReachedLimit, res.locals.ip)) { -// logger.error(`[RateLimit] for valid requests reached ${res.locals.ip}, ${req.get('User-Agent')}`); -// ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: false }; -// } -// res.status(options.statusCode).send(options.message); -// } -// }) -// } -// } - - +import { create as createError } from '@src/middleware/error'; +import { baseSlowDown, errorRateLimiter } from '@src/middleware/limit'; // example call: /write?user=xx&lat=00.000&lon=00.000×tamp=1704063600000&hdop=0.0&altitude=0.000&speed=0.000&heading=000.0 function errorChecking(req: Request, res: Response, next: NextFunction) { const errors = validationResult(req); if (!errors.isEmpty()) { - // if errors happend, then rateLimit to prevent key bruteforcing errorRateLimiter(req, res, () => { const errorAsJson = { errors: errors.array() }; @@ -101,7 +43,7 @@ async function writeData(req: Request, res: Response, next: NextFunction) { const router = express.Router(); -router.get('/', slowDown(baseSlowDown), entry.validate, errorChecking, writeData); -router.head('/', slowDown(baseSlowDown), entry.validate, errorChecking); + router.get('/', baseSlowDown, entry.validate, errorChecking, writeData); +router.head('/', baseSlowDown, entry.validate, errorChecking); export default router; \ No newline at end of file diff --git a/src/cache.ts b/src/middleware/cache.ts similarity index 100% rename from src/cache.ts rename to src/middleware/cache.ts diff --git a/src/error.ts b/src/middleware/error.ts similarity index 100% rename from src/error.ts rename to src/middleware/error.ts diff --git a/src/middleware/limit.ts b/src/middleware/limit.ts new file mode 100644 index 0000000..b7708cc --- /dev/null +++ b/src/middleware/limit.ts @@ -0,0 +1,44 @@ +import { Request, Response, NextFunction } from 'express'; +import { rateLimit, Options as rateLimiterOptions } from 'express-rate-limit'; +import { slowDown, Options as slowDownOptions } from 'express-slow-down'; +import logger from '@src/scripts/logger'; + + +// TODO clean up after 1 day? +const ipsThatReachedLimit: RateLimit.obj = {}; + +const baseOptions: Partial = { + windowMs: 5 * 60 * 1000, + //skip: (req, res) => (res.locals.ip == process.env.LOCALHOST) +} + +const baseSlowDownOptions: Partial = { + ...baseOptions, + delayAfter: 3, // no delay for amount of attempts + delayMs: (used: number) => (used - 3) * 125, // Add delay after delayAfter is reached +} + +const baseRateLimitOptions: Partial = { + ...baseOptions, + limit: 10, // Limit each IP per window + standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers + legacyHeaders: false, // Disable the `X-RateLimit-*` headers +} + + +/* +** exported section +*/ +export const baseSlowDown = slowDown(baseSlowDownOptions); + +export const errorRateLimiter = rateLimit({ + ...baseRateLimitOptions, + message: 'Too many requests with errors', + handler: (req: Request, res: Response, next: NextFunction, options: rateLimiterOptions) => { + if (!Object.prototype.hasOwnProperty.call(ipsThatReachedLimit, res.locals.ip)) { + logger.error(`[RateLimit] for invalid requests reached ${res.locals.ip}, ${req.get('User-Agent')}`); + ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: true }; + } + res.status(options.statusCode).send(options.message); + } +}); \ No newline at end of file diff --git a/src/models/entry.ts b/src/models/entry.ts index 4305fea..1f510bf 100644 --- a/src/models/entry.ts +++ b/src/models/entry.ts @@ -1,7 +1,7 @@ import { NextFunction, Request, Response } from 'express'; import { checkExact, query } from 'express-validator'; import { crypt } from '@src/scripts/crypt'; -import { create as createError } from '@src/error'; +import { create as createError } from '@src/middleware/error'; import * as file from '@src/scripts/file'; import { getTime } from '@src/scripts/time'; import { getSpeed } from '@src/scripts/speed'; diff --git a/src/scripts/file.ts b/src/scripts/file.ts index 5fdec3f..a8b693b 100644 --- a/src/scripts/file.ts +++ b/src/scripts/file.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; import { promisify } from 'util'; -import { create as createError } from '@src/error'; +import { create as createError } from '@src/middleware/error'; import { NextFunction, Response } from 'express'; import logger from '@src/scripts/logger'; From f40a710989ebcf728c30d106d14e9bbf0bc7d49d Mon Sep 17 00:00:00 2001 From: Type-Style Date: Tue, 13 Feb 2024 13:15:38 +0100 Subject: [PATCH 13/14] [Task] #41 ratelimiter cleaning up periodicly --- src/middleware/limit.ts | 25 ++++++++++++++++++++----- types.d.ts | 3 ++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/middleware/limit.ts b/src/middleware/limit.ts index b7708cc..e6596e4 100644 --- a/src/middleware/limit.ts +++ b/src/middleware/limit.ts @@ -4,12 +4,13 @@ import { slowDown, Options as slowDownOptions } from 'express-slow-down'; import logger from '@src/scripts/logger'; -// TODO clean up after 1 day? -const ipsThatReachedLimit: RateLimit.obj = {}; +/* +** configurations +*/ const baseOptions: Partial = { - windowMs: 5 * 60 * 1000, - //skip: (req, res) => (res.locals.ip == process.env.LOCALHOST) + windowMs: 30 * 60 * 1000, + skip: (req, res) => (res.locals.ip == process.env.LOCALHOST) } const baseSlowDownOptions: Partial = { @@ -26,6 +27,20 @@ const baseRateLimitOptions: Partial = { } +/* +** cleanup +*/ +const ipsThatReachedLimit: RateLimit.obj = {}; // prevent logs from flooding +setInterval(() => { + const oneHourAgo = Date.now() - 60 * 60 * 1000; + for (const ip in ipsThatReachedLimit) { + if (ipsThatReachedLimit[ip].time < oneHourAgo) { + delete ipsThatReachedLimit[ip]; + } + } +}, 60 * 60 * 1000); + + /* ** exported section */ @@ -37,7 +52,7 @@ export const errorRateLimiter = rateLimit({ handler: (req: Request, res: Response, next: NextFunction, options: rateLimiterOptions) => { if (!Object.prototype.hasOwnProperty.call(ipsThatReachedLimit, res.locals.ip)) { logger.error(`[RateLimit] for invalid requests reached ${res.locals.ip}, ${req.get('User-Agent')}`); - ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: true }; + ipsThatReachedLimit[res.locals.ip] = { limitReachedOnError: true, time: Date.now() }; } res.status(options.statusCode).send(options.message); } diff --git a/types.d.ts b/types.d.ts index 064db1f..8931ce5 100644 --- a/types.d.ts +++ b/types.d.ts @@ -4,7 +4,8 @@ namespace RateLimit { interface obj { [key: string]: { - limitReachedOnError: boolean + limitReachedOnError: boolean, + time: number } } } From 0dbadbb963fc4dec2a41d70bf4e24aabc9552d76 Mon Sep 17 00:00:00 2001 From: Type-Style Date: Tue, 13 Feb 2024 13:55:16 +0100 Subject: [PATCH 14/14] [Task] #41 skip tests in rateLimiting --- .github/workflows/main.yml | 2 +- src/app.ts | 1 + src/middleware/limit.ts | 2 +- src/tests/integration.test.ts | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a755f3f..5eaaa85 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: - name: Start server run: | sudo npm start & - sleep 6 # Give server some time to start + sleep 8 # Give server some time to start - name: Check if server is running run: | curl --fail http://localhost:80 || exit 1 diff --git a/src/app.ts b/src/app.ts index 3761d0b..1e59d96 100644 --- a/src/app.ts +++ b/src/app.ts @@ -69,6 +69,7 @@ app.use(function (req, res, next) { // limit request size limit when recieving d // routes app.get('/', (req, res) => { + console.log(req.ip + " - " + res.locals.ip); res.send('Hello World, via TypeScript and Node.js! ' + res.locals.ip); }); diff --git a/src/middleware/limit.ts b/src/middleware/limit.ts index e6596e4..a73e6c8 100644 --- a/src/middleware/limit.ts +++ b/src/middleware/limit.ts @@ -10,7 +10,7 @@ import logger from '@src/scripts/logger'; const baseOptions: Partial = { windowMs: 30 * 60 * 1000, - skip: (req, res) => (res.locals.ip == process.env.LOCALHOST) + skip: (req, res) => (res.locals.ip == "127.0.0.1" || res.locals.ip == "::1") } const baseSlowDownOptions: Partial = { diff --git a/src/tests/integration.test.ts b/src/tests/integration.test.ts index 2d40ced..6133c1d 100644 --- a/src/tests/integration.test.ts +++ b/src/tests/integration.test.ts @@ -129,7 +129,7 @@ describe("GET /write", () => { expect(entry.time.created).toBeGreaterThan(date.getTime()); expect(entry.time.diff).toBeGreaterThan(3.5); - expect(entry.time.diff).toBeLessThan(4); + expect(entry.time.diff).toBeLessThan(4.6); const germanDayPattern = "(Montag|Dienstag|Mittwoch|Donnerstag|Freitag|Samstag|Sonntag)";