fix: remove external URLs from addFromURL tests

Relying on external URLs makes our tests dependent on the availability and response time of these services as well as not being able to run without an internet connection. This PR removes those requests to external URLs and instead spins up test HTTP(s) servers as they are needed to serve the content that `ipfs.util.addFromURL` consumes.

resolves #803

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

test/fixtures/ssl/cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOzCCAiMCCQCVqVeRIp9pFDANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJV
+UzENMAsGA1UECAwEVXRhaDEOMAwGA1UEBwwFUHJvdm8xIzAhBgNVBAoMGkFDTUUg
+U2lnbmluZyBBdXRob3JpdHkgSW5jMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0x
+ODA4MTQyMDEzNTdaFw0xOTEyMjcyMDEzNTdaMFgxCzAJBgNVBAYTAlVTMQ0wCwYD
+VQQIDARVdGFoMQ4wDAYDVQQHDAVQcm92bzEWMBQGA1UECgwNQUNNRSBUZWNoIElu
+YzESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3gb7sO96P
+tOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731vRZyo0kx
+bSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0Oy0fC6Of
+VWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN59uq+JqUu
+8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT+3PDGU8c
+oBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQCr
+R7eZxicHjJoRcbsPBDQpzx9uSux3uvpN93pyJrXXHiil/5SE7CYeDqv5+nV2p6HA
+6KONUAmpId0iHAEi9u+0/LgPWyYQMzT3sfBhkO8RRaMYI87VuKbk5PFmlZbD843+
+Qmg3Se2F7BDnTf88xA6QWR4DCejy+ZHfDRFrh3xfFl4tX1UNgqiTGfjPCzblhWx9
+ygzlT+flN2j3NkAlhUEV89pnH4EQWILePMTT4wh2XOQj1VFJ+2ATojHFVUTtNWAJ
+xrY/Q9cMYsZ++I8i9bHMZoyc1bSUd5CNFpQdfjVzlgMPT9Jj/fzWIQz+wq0KeRLI
+dLWsa2MZr0GZnTU39YwH
+-----END CERTIFICATE-----

test/fixtures/ssl/privkey.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3
+gb7sO96PtOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731
+vRZyo0kxbSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0
+Oy0fC6OfVWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN5
+9uq+JqUu8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT
++3PDGU8coBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABAoIBAH5fbfFqOpie6T8T
+wj4bTGbA4bsZkD9JeU7ZiXubA/ABd5xyduwky2JugH0vrvRC3IVrE0qU8OiBA7Le
+/EUx5/kRSPFsZBf/wwChRiB4WlYsvllLZ76nRxyepZNN7H5dx3Hkk1gjVREi71jd
+ATUtGxfsRG77DV5WbcshIlLLhT9iaohsalmClAFBmwhqnRMvOXHiQyRbvB0fOX08
+uVlObOqo9jLB8N5C/ux+wFEP4wi/AxVqs9ih7Ss7T7+pmOCVWhOnbYcoY2jdaJ11
+iLK4F3rv/jQ82OwUpzrWsPedmZUzlOO8xdV3b8hOcPHs/BKvYed7aHSn6b5eVKKT
+zT8vQoECgYEA+K9pvw9K/7+F810MHG+nZ0gtVWmXJp49jB7zQ6QMIex2sUajY2y9
+bEJX8T6rdu3qd+HYU4zl3qt+MUnsVQEBNkLPAn3od0qIWXxu1SL2GF8SDV1xJWK1
+Fp0YDe9blaz1JsmSgieNcSoSwqE2V97Wfd/m+EUfyhQt9HX55H5UgAUCgYEA8gkW
+0xZKuHhBMYpcES2P5H5q6HN2fcEQycMuS3agAOhrFPYUT1DVNhbfNVmbOvL2NRWI
+hXixo5SkuEuq2fjmEoeLPTmeKO5LM4IVqovWCYomSftKDpzw4HRn2jvKzi2+mg8J
+qktIMqRqHu/O1NUIsszCIu4c5DzUdhr4N7GXOaECgYAEd1oF1Wd6al0kfsJN7G9s
+Om6d/xR43BSs5I1n5JVXMqD7FBKxIW3ReOuNaJu5uhIg7wxsi7ZBJoFQr0wwRqFX
+8SE4oTxAkDUcrlBrQYJ785Embkwu6LPp4Q5iia7yZDXO6YXZEo7GvoOxvSV1tInT
+nubOBKfKgExG/KttQBuSZQKBgAzYOqPdLP35M8yDQTuQJXDE3LuVVRZ7Zn6uowhS
+NU+XBgfIv28uJQKH2DSmmrxYJITQrbwXmaXKv6sgKOMEeIFHPDZ1llUpwEftgWTZ
+ovRCpqGKenWoEoh25QQJ5Eto1hKq9aJZ+GznmNIne9yDqcCDaVIdPN9H8yaJa97Y
+x+PBAoGAOiK6xAbPyJSKDSTGZzdv8+yeOdNeJjRHxKJs+4YsDchmdumrqow83DBP
+7ulIJD9pcmsWj+8fntMcsTX5mvzJd5LsKc7Maa5/LtitsLsynu78QFg4Njj8sAKn
+3991i8J98DZ9zqmkxJJhGwstCHG+c+Q7lA6kZ1UdbWJwYwIHjAs=
+-----END RSA PRIVATE KEY-----

test/util.spec.js

@@ -10,6 +10,11 @@ const isNode = require('detect-node')
 const path = require('path')
 const fs = require('fs')
 const os = require('os')
+const http = require('http')
+const https = require('https')
+const each = require('async/each')
+const waterfall = require('async/waterfall')
+const parallel = require('async/parallel')
 
 const IPFSApi = require('../src')
 const f = require('./utils/factory')
@@ -32,7 +37,8 @@ describe('.util', () => {
     })
   })
 
-  after((done) => {
+  after(function (done) {
+    this.timeout(10 * 1000)
     if (!ipfsd) return done()
     ipfsd.stop(done)
   })
@@ -113,81 +119,243 @@ describe('.util', () => {
   })
 
   describe('.urlAdd', () => {
-    it('http', function (done) {
-      this.timeout(40 * 1000)
+    let testServers = []
+
+    const sslOpts = {
+      key: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'privkey.pem')),
+      cert: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'cert.pem'))
+    }
+
+    const startTestServer = (handler, opts, cb) => {
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      const agent = opts.secure ? https : http
+      const serverOpts = opts.secure ? sslOpts : {}
+      const server = agent.createServer(serverOpts, handler)
+
+      server.listen((err) => {
+        if (err) return cb(err)
+        testServers.push(server)
+        cb(null, server)
+      })
+    }
 
-      ipfs.util.addFromURL('http://example.com/', (err, result) => {
-        expect(err).to.not.exist()
-        expect(result.length).to.equal(1)
-        done()
+    beforeEach(() => {
+      // Instructs node to not reject our snake oil SSL certificate when it
+      // can't verify the cerificate authority
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0
+    })
+
+    afterEach((done) => {
+      // Reinstate unauthorised SSL cert rejection
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1
+
+      each(testServers, (server, cb) => server.close(cb), (err) => {
+        testServers = []
+        done(err)
       })
     })
 
-    it('https', function (done) {
-      this.timeout(40 * 1000)
+    it('http', (done) => {
+      const handler = (req, res) => {
+        res.write(`TEST${Date.now()}`)
+        res.end()
+      }
 
-      ipfs.util.addFromURL('https://example.com/', (err, result) => {
+      startTestServer(handler, (err, server) => {
         expect(err).to.not.exist()
-        expect(result.length).to.equal(1)
-        done()
+
+        const url = `http://127.0.0.1:${server.address().port}/`
+        ipfs.util.addFromURL(url, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result.length).to.equal(1)
+          done()
+        })
       })
     })
 
-    it('http with redirection', function (done) {
-      this.timeout(40 * 1000)
+    it('https', (done) => {
+      const handler = (req, res) => {
+        res.write(`TEST${Date.now()}`)
+        res.end()
+      }
 
-      ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => {
+      startTestServer(handler, { secure: true }, (err, server) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
-        done()
+
+        const url = `https://127.0.0.1:${server.address().port}/`
+        ipfs.util.addFromURL(url, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result.length).to.equal(1)
+          done()
+        })
       })
     })
 
-    it('https with redirection', function (done) {
-      this.timeout(40 * 1000)
-
-      ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
+    it('http with redirection', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      waterfall([
+        (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, cb)
+        },
+        (serverA, cb) => {
+          const url = `http://127.0.0.1:${serverA.address().port}`
+          const handler = (req, res) => {
+            res.statusCode = 302
+            res.setHeader('Location', url)
+            res.end()
+          }
+          startTestServer(handler, (err, serverB) => {
+            if (err) return cb(err)
+            cb(null, { a: serverA, b: serverB })
+          })
+        }
+      ], (err, servers) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
-        done()
+
+        ipfs.add(data, (err, res) => {
+          expect(err).to.not.exist()
+
+          const expectedHash = res[0].hash
+          const url = `http://127.0.0.1:${servers.b.address().port}`
+
+          ipfs.util.addFromURL(url, (err, result) => {
+            expect(err).to.not.exist()
+            expect(result[0].hash).to.equal(expectedHash)
+            done()
+          })
+        })
       })
     })
 
-    it('with only-hash=true', function () {
-      this.timeout(40 * 1000)
+    it('https with redirection', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      waterfall([
+        (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, cb)
+        },
+        (serverA, cb) => {
+          const url = `https://127.0.0.1:${serverA.address().port}`
+          const handler = (req, res) => {
+            res.statusCode = 302
+            res.setHeader('Location', url)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, (err, serverB) => {
+            if (err) return cb(err)
+            cb(null, { a: serverA, b: serverB })
+          })
+        }
+      ], (err, servers) => {
+        expect(err).to.not.exist()
 
-      return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true })
-        .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000))
+        ipfs.add(data, (err, res) => {
+          expect(err).to.not.exist()
+
+          const expectedHash = res[0].hash
+          const url = `https://127.0.0.1:${servers.b.address().port}`
+
+          ipfs.util.addFromURL(url, (err, result) => {
+            expect(err).to.not.exist()
+            expect(result[0].hash).to.equal(expectedHash)
+            done()
+          })
+        })
+      })
     })
 
-    it('with wrap-with-directory=true', function (done) {
-      this.timeout(40 * 1000)
+    it('with only-hash=true', (done) => {
+      const handler = (req, res) => {
+        res.write(`TEST${Date.now()}`)
+        res.end()
+      }
 
-      ipfs.util.addFromURL('http://ipfs.io/ipfs/QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61/969165.jpg?foo=bar#buzz', {
-        wrapWithDirectory: true
-      }, (err, result) => {
+      startTestServer(handler, (err, server) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
-        expect(result[0].path).to.equal('969165.jpg')
-        expect(result[1].hash).to.equal('QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61')
-        expect(result.length).to.equal(2)
-        done()
+
+        const url = `http://127.0.0.1:${server.address().port}/`
+
+        ipfs.util.addFromURL(url, { onlyHash: true }, (err, res) => {
+          expect(err).to.not.exist()
+
+          // A successful object.get for this size data took my laptop ~14ms
+          let didTimeout = false
+          const timeoutId = setTimeout(() => {
+            didTimeout = true
+            done()
+          }, 500)
+
+          ipfs.object.get(res[0].hash, () => {
+            clearTimeout(timeoutId)
+            if (didTimeout) return
+            expect(new Error('did not timeout')).to.not.exist()
+          })
+        })
       })
     })
 
-    it('with wrap-with-directory=true and URL-escaped file name', function (done) {
-      this.timeout(40 * 1000)
+    it('with wrap-with-directory=true', (done) => {
+      const filename = `TEST${Date.now()}.txt`
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => startTestServer((req, res) => {
+          res.write(data)
+          res.end()
+        }, cb),
+        expectedResult: (cb) => {
+          ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb)
+        }
+      }, (err, taskResult) => {
+        expect(err).to.not.exist()
+
+        const { server, expectedResult } = taskResult
+        const url = `http://127.0.0.1:${server.address().port}/${filename}?foo=bar#buzz`
+
+        ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
+      })
+    })
 
-      // Sample URL contains URL-escaped ( ) and local diacritics
-      ipfs.util.addFromURL('https://upload.wikimedia.org/wikipedia/commons/thumb/c/cf/Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg/320px-Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg?foo=bar#buzz', {
-        wrapWithDirectory: true
-      }, (err, result) => {
+    it('with wrap-with-directory=true and URL-escaped file name', (done) => {
+      const filename = '320px-Domažlice,_Jiráskova_43_(9102).jpg'
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => startTestServer((req, res) => {
+          res.write(data)
+          res.end()
+        }, cb),
+        expectedResult: (cb) => {
+          ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb)
+        }
+      }, (err, taskResult) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmRJ9ExxSMV4BLF9ZJUb2mLngupm6BXZEek755VHGTJo2Y')
-        expect(result[0].path).to.equal('320px-Domažlice,_Jiráskova_43_(9102).jpg')
-        expect(result[1].hash).to.equal('QmbxsHFU3sCfr8wszDHuDLA76C2xCv9HT8L3aC1pBwgaHk')
-        expect(result.length).to.equal(2)
-        done()
+
+        const { server, expectedResult } = taskResult
+        const url = `http://127.0.0.1:${server.address().port}/${encodeURIComponent(filename)}?foo=bar#buzz`
+
+        ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
       })
     })