Merge pull request #600 from oceanprotocol/feature/url_protection

add unsafe URL & make config required for Storage classes

env.md

@@ -29,6 +29,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `LOG_CONSOLE`: Write logs to the console. Default is `false`, but becomes `true` if neither `LOG_FILES` or `LOG_DB` are set.
 - `LOG_FILES`: Write logs to files. Default is `false`
 - `LOG_DB`: Write logs to noSQL database. Default is `false`
+- `UNSAFE_URLS`: Array or regular expression URLs to be excluded from access.Example: ["^.*(169.254.169.254).*","^.*(127.0.0.1).*"]
 
 ## HTTP
 

src/@types/OceanNode.ts

@@ -75,6 +75,7 @@ export interface OceanNodeConfig {
   codeHash?: string
   rateLimit?: number
   denyList?: DenyList
+  unsafeURLs?: string[]
 }
 
 export interface P2PStatusResponse {

src/components/core/handler/downloadHandler.ts

@@ -65,10 +65,10 @@ export async function handleDownloadUrlCommand(
 ): Promise<P2PCommandResponse> {
   const encryptFile = !!task.aes_encrypted_key
   CORE_LOGGER.logMessage('DownloadCommand requires file encryption? ' + encryptFile, true)
-
+  const config = await getConfiguration()
   try {
     // Determine the type of storage and get a readable stream
-    const storage = Storage.getStorageClass(task.fileObject)
+    const storage = Storage.getStorageClass(task.fileObject, config)
     if (
       storage instanceof ArweaveStorage &&
       !existsEnvironmentVariable(ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY)
@@ -125,7 +125,6 @@ export async function handleDownloadUrlCommand(
       // we parse the string into the object again
       const encryptedObject = ethCrypto.cipher.parse(task.aes_encrypted_key)
       // get the key from configuration
-      const config = await getConfiguration()
       const nodePrivateKey = Buffer.from(config.keys.privateKey).toString('hex')
       const decrypted = await ethCrypto.decryptWithPrivateKey(
         nodePrivateKey,

src/components/core/handler/encryptHandler.ts

@@ -119,7 +119,7 @@ export class EncryptFileHandler extends Handler {
       }
       let encryptedContent: Buffer
       if (task.files) {
-        const storage = Storage.getStorageClass(task.files)
+        const storage = Storage.getStorageClass(task.files, config)
         encryptedContent = await storage.encryptContent(task.encryptionType)
       } else if (task.rawData !== null) {
         encryptedContent = await encrypt(task.rawData, task.encryptionType)

src/components/core/handler/fileInfoHandler.ts

@@ -8,7 +8,7 @@ import {
 } from '../../../@types/fileObject.js'
 import { FileInfoCommand } from '../../../@types/commands.js'
 import { CORE_LOGGER } from '../../../utils/logging/common.js'
-import { ArweaveStorage, IpfsStorage, UrlStorage } from '../../storage/index.js'
+import { Storage } from '../../storage/index.js'
 import { Handler } from './handler.js'
 import { validateDDOIdentifier } from './ddoHandler.js'
 import { fetchFileMetadata } from '../../../utils/asset.js'
@@ -18,7 +18,7 @@ import {
   validateCommandParameters
 } from '../../httpRoutes/validateCommands.js'
 import { getFile } from '../../../utils/file.js'
-
+import { getConfiguration } from '../../../utils/index.js'
 async function formatMetadata(file: ArweaveFileObject | IpfsFileObject | UrlFileObject) {
   const url =
     file.type === 'url'
@@ -79,14 +79,7 @@ export class FileInfoHandler extends Handler {
       let fileInfo = []
 
       if (task.file && task.type) {
-        const storage =
-          task.type === 'url'
-            ? new UrlStorage(task.file as UrlFileObject)
-            : task.type === 'arweave'
-            ? new ArweaveStorage(task.file as ArweaveFileObject)
-            : task.type === 'ipfs'
-            ? new IpfsStorage(task.file as IpfsFileObject)
-            : null
+        const storage = Storage.getStorageClass(task.file, await getConfiguration())
 
         fileInfo = await storage.getFileInfo({
           type: task.type,

src/components/storage/index.ts

@@ -8,19 +8,23 @@ import {
   UrlFileObject,
   EncryptMethod
 } from '../../@types/fileObject.js'
+import { OceanNodeConfig } from '../../@types/OceanNode.js'
 import { fetchFileMetadata } from '../../utils/asset.js'
 import axios from 'axios'
 import urlJoin from 'url-join'
 import { encrypt as encryptData, decrypt as decryptData } from '../../utils/crypt.js'
 import { Readable } from 'stream'
-import { getConfiguration } from '../../utils/index.js'
 import { CORE_LOGGER } from '../../utils/logging/common.js'
 
 export abstract class Storage {
   private file: UrlFileObject | IpfsFileObject | ArweaveFileObject
-
-  public constructor(file: UrlFileObject | IpfsFileObject | ArweaveFileObject) {
+  config: OceanNodeConfig
+  public constructor(
+    file: UrlFileObject | IpfsFileObject | ArweaveFileObject,
+    config: OceanNodeConfig
+  ) {
     this.file = file
+    this.config = config
   }
 
   abstract validate(): [boolean, string]
@@ -54,17 +58,20 @@ export abstract class Storage {
     }
   }
 
-  static getStorageClass(file: any): UrlStorage | IpfsStorage | ArweaveStorage {
+  static getStorageClass(
+    file: any,
+    config: OceanNodeConfig
+  ): UrlStorage | IpfsStorage | ArweaveStorage {
     const { type } = file
     switch (
       type?.toLowerCase() // case insensitive
     ) {
       case FileObjectType.URL:
-        return new UrlStorage(file)
+        return new UrlStorage(file, config)
       case FileObjectType.IPFS:
-        return new IpfsStorage(file)
+        return new IpfsStorage(file, config)
       case FileObjectType.ARWEAVE:
-        return new ArweaveStorage(file)
+        return new ArweaveStorage(file, config)
       default:
         throw new Error(`Invalid storage type: ${type}`)
     }
@@ -118,7 +125,7 @@ export abstract class Storage {
   }
 
   async decrypt() {
-    const { keys } = await getConfiguration()
+    const { keys } = this.config
     const nodeId = keys.peerId.toString()
 
     if (!this.canDecrypt(nodeId)) {
@@ -173,8 +180,8 @@ export abstract class Storage {
 }
 
 export class UrlStorage extends Storage {
-  public constructor(file: UrlFileObject) {
-    super(file)
+  public constructor(file: UrlFileObject, config: OceanNodeConfig) {
+    super(file, config)
     const [isValid, message] = this.validate()
     if (isValid === false) {
       throw new Error(`Error validationg the URL file: ${message}`)
@@ -189,6 +196,17 @@ export class UrlStorage extends Storage {
     if (!['get', 'post'].includes(file.method?.toLowerCase())) {
       return [false, 'Invalid method for URL']
     }
+    if (this.config && this.config.unsafeURLs) {
+      for (const regex of this.config.unsafeURLs) {
+        try {
+          // eslint-disable-next-line security/detect-non-literal-regexp
+          const pattern = new RegExp(regex)
+          if (pattern.test(file.url)) {
+            return [false, 'URL is marked as unsafe']
+          }
+        } catch (e) {}
+      }
+    }
     if (this.isFilePath() === true) {
       return [false, 'URL looks like a file path']
     }
@@ -248,8 +266,8 @@ export class UrlStorage extends Storage {
 }
 
 export class ArweaveStorage extends Storage {
-  public constructor(file: ArweaveFileObject) {
-    super(file)
+  public constructor(file: ArweaveFileObject, config: OceanNodeConfig) {
+    super(file, config)
 
     const [isValid, message] = this.validate()
     if (isValid === false) {
@@ -326,8 +344,8 @@ export class ArweaveStorage extends Storage {
 }
 
 export class IpfsStorage extends Storage {
-  public constructor(file: IpfsFileObject) {
-    super(file)
+  public constructor(file: IpfsFileObject, config: OceanNodeConfig) {
+    super(file, config)
 
     const [isValid, message] = this.validate()
     if (isValid === false) {