41 add rate limiter (#45)

* [Task] #18, limit request size for security reasons

* [Task] #43, introduce gzip to transfer data

* [Task] #34 improve error handling, log server shutdowns

* [Task] #34 installed and integrated tooBusy to send 503 when load is high

* [Task] #34 improved tooBusy, improved formatting

* [Task, Temp] #41 installed ratelimiter and slowDown

* [Task] #42 cleanup ipv6 addresses

* [Change] #10 error handling for better gitBash and txt output, also reduced stack in case of validation errors

* [Task] #41 prepare Log for RateLImit errors

* [Temp] #41 write route rateLImited

temp: see Todos

* [Task] #34 colorize prefix in console

* [Task] #42 extract middlewares and move to folder

* [Task] #41 ratelimiter cleaning up periodicly

* [Task] #41 skip tests in rateLimiting

.github/workflows/main.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Start server
       run: |
         sudo npm start &
-        sleep 6 # Give server some time to start
+        sleep 8 # Give server some time to start
     - name: Check if server is running
       run: |
         curl --fail http://localhost:80 || exit 1

package.json

@@ -18,11 +18,13 @@
     "@jest/globals": "^29.7.0",
     "@tsconfig/node20": "^20.1.2",
     "@types/bcrypt": "^5.0.2",
+    "@types/compression": "^1.7.5",
     "@types/dotenv": "^8.2.0",
     "@types/express": "^4.17.21",
     "@types/hpp": "^0.2.5",
     "@types/jest": "^29.5.11",
     "@types/node": "^20.10.6",
+    "@types/toobusy-js": "^0.5.4",
     "@typescript-eslint/eslint-plugin": "^6.18.1",
     "@typescript-eslint/parser": "^6.18.1",
     "axios": "^1.6.5",
@@ -37,11 +39,16 @@
   },
   "dependencies": {
     "chalk": "^4.1.2",
+    "compression": "^1.7.4",
     "express": "^4.18.2",
+    "express-rate-limit": "^7.1.5",
+    "express-slow-down": "^2.0.1",
     "express-validator": "^7.0.1",
     "helmet": "^7.1.0",
     "hpp": "^0.2.3",
-    "module-alias": "^2.2.3"
+    "module-alias": "^2.2.3",
+    "raw-body": "^2.5.2",
+    "toobusy-js": "^0.5.1"
   },
   "_moduleAliases": {
     "@src": "dist"

src/app.ts

@@ -1,58 +1,117 @@
 require('module-alias/register');
 import { config } from 'dotenv';
 import express from 'express';
+import toobusy from 'toobusy-js';
+// import { rateLimit } from 'express-rate-limit';
+// import { slowDown } from 'express-slow-down';
+import compression from 'compression';
 import helmet from 'helmet';
 import hpp from 'hpp';
-import cache from './cache';
-import * as error from "./error";
+import getRawBody from 'raw-body';
+import cache from './middleware/cache';
+import * as error from "./middleware/error";
 import writeRouter from '@src/controller/write';
 import readRouter from '@src/controller/read';
-import path  from 'path';
+import path from 'path';
 import logger from '@src/scripts/logger';
 
+// console.log({ "status": 403, "name": "Error", "message": { "errors": [{ "type": "field", "msg": "Invalid value", "path": "user", "location": "query" }, { "type": "field", "msg": "is required", "path": "lat", "location": "query" }]}});
+// console.log(JSON.stringify({ "status": 403, "name": "Error", "message": { "errors": [{ "type": "field", "msg": "Invalid value", "path": "user", "location": "query" }, { "type": "field", "msg": "is required", "path": "lat", "location": "query" }]}}, null, 2));
+
 // configurations
-config();
+config(); // dotenv
+
 const app = express();
-app.use(
-  helmet({
-    contentSecurityPolicy: {
-      directives: {
-        "default-src": "'self'",        
-        "img-src": "*"
-      }
-    }
-  })
-);
 
-app.use(hpp());
+app.use((req, res, next) => { // monitor eventloop to block requests if busy
+  if (toobusy()) {
+    res.status(503).set({ 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Retry-After': '60' }).send("I'm busy right now, sorry.");
+  } else { next(); }
+});
+app.use((req, res, next) => { // clean up IPv6 Addresses
+  if (req.ip) {
+    res.locals.ip = req.ip.startsWith('::ffff:') ? req.ip.substring(7) : req.ip;
+    next();
+  } else {
+    const message = "No IP provided"
+    logger.error(message);
+    res.status(400).send(message);
+  }
+
+})
+
+// const slowDownLimiter = slowDown({
+// 	windowMs: 1 * 60 * 1000,
+// 	delayAfter: 5, // Allow 5 requests per 15 minutes.
+// 	delayMs: (used) => (used - 5) * 1000, // Add delay after delayAfter is reached
+// })
+
+// const rateLimiter = rateLimit({
+//   windowMs: 1 * 60 * 1000,
+//   limit: 10, // Limit each IP per `window`
+//   standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
+//   legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+// })
+
+app.use(helmet({ contentSecurityPolicy: { directives: { "default-src": "'self'", "img-src": "*" } } }));
 app.use(cache);
+app.use(compression())
+app.use(hpp());
+app.use(function (req, res, next) { // limit request size limit when recieving data
+  if (!['POST', 'PUT', 'DELETE'].includes(req.method)) { return next(); }
+  getRawBody(req, { length: req.headers['content-length'], limit: '1mb', encoding: true },
+    function (err) {
+      if (err) { return next(err) }
+      next()
+    }
+  )
+})
 
 // routes
 app.get('/', (req, res) => {
-  res.send('Hello World, via TypeScript and Node.js!');  
+  console.log(req.ip + " - " + res.locals.ip);
+  res.send('Hello World, via TypeScript and Node.js! ' + res.locals.ip);
 });
 
-
 app.use('/write', writeRouter);
 app.use('/read', readRouter);
 
 // use httpdocs as static folder
 app.use('/', express.static(path.join(__dirname, 'httpdocs'), {
   extensions: ['html', 'txt', "pdf"],
-  index: "start.html", 
-}))
+  index: ["start.html", "start.txt"],
+}));
 
 // error handling
 app.use(error.notFound);
 app.use(error.handler);
 
 // init server
-app.listen(80, () => {
-  logger.log(`Server running //localhost:80, ENV: ${process.env.NODE_ENV}`, true); 
+const server = app.listen(80, () => {
+  logger.log(`Server running //localhost:80, ENV: ${process.env.NODE_ENV}`, true);
+});
+
+// catching shutdowns
+['SIGINT', 'SIGTERM', 'exit'].forEach((signal) => {
+  process.on(signal, () => {
+    function logAndExit() {
+      // calling .shutdown allows your process to exit normally
+      toobusy.shutdown();
+      logger.log(`Server shutdown on signal: ${signal} //localhost:80`, true);
+      process.exit();
+    }
+    if (signal != "exit") { // give the server time to shutdown before closing
+      server.close(logAndExit);
+    } else {
+      logger.log(`Server shutdown immediate: ${signal} //localhost:80`, true);
+    }
+  });
 });
 
-process.on('uncaughtException', function(err) {
+// last resort error handling
+process.on('uncaughtException', function (err) {
   console.error('Caught exception:', err);
   logger.error(err);
+  server.close();
   process.exit(1);
 });

src/controller/read.ts

@@ -1,6 +1,6 @@
 import express, { Request, Response, NextFunction } from 'express';
 import * as file from '@src/scripts/file';
-import { create as createError } from '@src/error';
+import { create as createError } from '@src/middleware/error';
 import { validationResult, query } from 'express-validator';
 
 const router = express.Router();