fix: remove external urls from addFromURL tests

test/fixtures/ssl/cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOzCCAiMCCQCVqVeRIp9pFDANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJV
+UzENMAsGA1UECAwEVXRhaDEOMAwGA1UEBwwFUHJvdm8xIzAhBgNVBAoMGkFDTUUg
+U2lnbmluZyBBdXRob3JpdHkgSW5jMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0x
+ODA4MTQyMDEzNTdaFw0xOTEyMjcyMDEzNTdaMFgxCzAJBgNVBAYTAlVTMQ0wCwYD
+VQQIDARVdGFoMQ4wDAYDVQQHDAVQcm92bzEWMBQGA1UECgwNQUNNRSBUZWNoIElu
+YzESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3gb7sO96P
+tOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731vRZyo0kx
+bSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0Oy0fC6Of
+VWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN59uq+JqUu
+8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT+3PDGU8c
+oBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQCr
+R7eZxicHjJoRcbsPBDQpzx9uSux3uvpN93pyJrXXHiil/5SE7CYeDqv5+nV2p6HA
+6KONUAmpId0iHAEi9u+0/LgPWyYQMzT3sfBhkO8RRaMYI87VuKbk5PFmlZbD843+
+Qmg3Se2F7BDnTf88xA6QWR4DCejy+ZHfDRFrh3xfFl4tX1UNgqiTGfjPCzblhWx9
+ygzlT+flN2j3NkAlhUEV89pnH4EQWILePMTT4wh2XOQj1VFJ+2ATojHFVUTtNWAJ
+xrY/Q9cMYsZ++I8i9bHMZoyc1bSUd5CNFpQdfjVzlgMPT9Jj/fzWIQz+wq0KeRLI
+dLWsa2MZr0GZnTU39YwH
+-----END CERTIFICATE-----

test/fixtures/ssl/privkey.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6x6mTXV+rC35QW/sPutT1O1cugtnw+UsJx7EGgzyjh7EoXE3
+gb7sO96PtOI5zknb0vecckbiVkesmLnAs2iNa1u9EiRr6WHdc+1MfUCxyHRfP731
+vRZyo0kxbSXerE0qZ2N3M1XyndZF7VMthKDKIg0ZR0TvdjwLqyLYEHAnRBhJLRS0
+Oy0fC6OfVWCO3gIuk1HkTXH+/ZMA/obqrtlisxY85mMdlRz+1PNdZBMf+NxmrXN5
+9uq+JqUu8/v1oQ8jH2iU9IWeqyawHDEvPW3aDorfaWGyats5Xd3cT2Ph4xF9tBLT
++3PDGU8coBmTHWDenYn+TCkCseayo1JCO5igJQIDAQABAoIBAH5fbfFqOpie6T8T
+wj4bTGbA4bsZkD9JeU7ZiXubA/ABd5xyduwky2JugH0vrvRC3IVrE0qU8OiBA7Le
+/EUx5/kRSPFsZBf/wwChRiB4WlYsvllLZ76nRxyepZNN7H5dx3Hkk1gjVREi71jd
+ATUtGxfsRG77DV5WbcshIlLLhT9iaohsalmClAFBmwhqnRMvOXHiQyRbvB0fOX08
+uVlObOqo9jLB8N5C/ux+wFEP4wi/AxVqs9ih7Ss7T7+pmOCVWhOnbYcoY2jdaJ11
+iLK4F3rv/jQ82OwUpzrWsPedmZUzlOO8xdV3b8hOcPHs/BKvYed7aHSn6b5eVKKT
+zT8vQoECgYEA+K9pvw9K/7+F810MHG+nZ0gtVWmXJp49jB7zQ6QMIex2sUajY2y9
+bEJX8T6rdu3qd+HYU4zl3qt+MUnsVQEBNkLPAn3od0qIWXxu1SL2GF8SDV1xJWK1
+Fp0YDe9blaz1JsmSgieNcSoSwqE2V97Wfd/m+EUfyhQt9HX55H5UgAUCgYEA8gkW
+0xZKuHhBMYpcES2P5H5q6HN2fcEQycMuS3agAOhrFPYUT1DVNhbfNVmbOvL2NRWI
+hXixo5SkuEuq2fjmEoeLPTmeKO5LM4IVqovWCYomSftKDpzw4HRn2jvKzi2+mg8J
+qktIMqRqHu/O1NUIsszCIu4c5DzUdhr4N7GXOaECgYAEd1oF1Wd6al0kfsJN7G9s
+Om6d/xR43BSs5I1n5JVXMqD7FBKxIW3ReOuNaJu5uhIg7wxsi7ZBJoFQr0wwRqFX
+8SE4oTxAkDUcrlBrQYJ785Embkwu6LPp4Q5iia7yZDXO6YXZEo7GvoOxvSV1tInT
+nubOBKfKgExG/KttQBuSZQKBgAzYOqPdLP35M8yDQTuQJXDE3LuVVRZ7Zn6uowhS
+NU+XBgfIv28uJQKH2DSmmrxYJITQrbwXmaXKv6sgKOMEeIFHPDZ1llUpwEftgWTZ
+ovRCpqGKenWoEoh25QQJ5Eto1hKq9aJZ+GznmNIne9yDqcCDaVIdPN9H8yaJa97Y
+x+PBAoGAOiK6xAbPyJSKDSTGZzdv8+yeOdNeJjRHxKJs+4YsDchmdumrqow83DBP
+7ulIJD9pcmsWj+8fntMcsTX5mvzJd5LsKc7Maa5/LtitsLsynu78QFg4Njj8sAKn
+3991i8J98DZ9zqmkxJJhGwstCHG+c+Q7lA6kZ1UdbWJwYwIHjAs=
+-----END RSA PRIVATE KEY-----

test/util.spec.js

@@ -10,6 +10,11 @@ const isNode = require('detect-node')
 const path = require('path')
 const fs = require('fs')
 const os = require('os')
+const http = require('http')
+const https = require('https')
+const each = require('async/each')
+const waterfall = require('async/waterfall')
+const parallel = require('async/parallel')
 
 const IPFSApi = require('../src')
 const f = require('./utils/factory')
@@ -32,7 +37,8 @@ describe('.util', () => {
     })
   })
 
-  after((done) => {
+  after(function (done) {
+    this.timeout(10 * 1000)
     if (!ipfsd) return done()
     ipfsd.stop(done)
   })
@@ -113,81 +119,257 @@ describe('.util', () => {
   })
 
   describe('.urlAdd', () => {
-    it('http', function (done) {
-      this.timeout(40 * 1000)
-
-      ipfs.util.addFromURL('http://example.com/', (err, result) => {
-        expect(err).to.not.exist()
-        expect(result.length).to.equal(1)
-        done()
+    let testServers = []
+
+    const sslOpts = {
+      key: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'privkey.pem')),
+      cert: fs.readFileSync(path.join(__dirname, 'fixtures', 'ssl', 'cert.pem'))
+    }
+
+    const startTestServer = (handler, opts, cb) => {
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      const server = opts.secure
+        ? https.createServer(sslOpts, handler)
+        : http.createServer(handler)
+
+      server.listen((err) => {
+        if (err) return cb(err)
+        testServers.push(server)
+        cb(null, server)
       })
+    }
+
+    beforeEach(() => {
+      // Instructs node to not reject our snake oil SSL certificate when it
+      // can't verify the certificate authority
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0
     })
 
-    it('https', function (done) {
-      this.timeout(40 * 1000)
+    afterEach((done) => {
+      // Reinstate unauthorised SSL cert rejection
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1
 
-      ipfs.util.addFromURL('https://example.com/', (err, result) => {
-        expect(err).to.not.exist()
-        expect(result.length).to.equal(1)
-        done()
+      each(testServers, (server, cb) => server.close(cb), (err) => {
+        testServers = []
+        done(err)
       })
     })
 
-    it('http with redirection', function (done) {
-      this.timeout(40 * 1000)
-
-      ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => {
+    it('http', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, cb)
+        },
+        expectedResult: (cb) => ipfs.add(data, cb)
+      }, (err, taskResult) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
-        done()
+        const { server, expectedResult } = taskResult
+
+        const url = `http://127.0.0.1:${server.address().port}/`
+        ipfs.util.addFromURL(url, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
       })
     })
 
-    it('https with redirection', function (done) {
-      this.timeout(40 * 1000)
+    it('https', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, cb)
+        },
+        expectedResult: (cb) => ipfs.add(data, cb)
+      }, (err, taskResult) => {
+        expect(err).to.not.exist()
+        const { server, expectedResult } = taskResult
 
-      ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
+        const url = `https://127.0.0.1:${server.address().port}/`
+        ipfs.util.addFromURL(url, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
+      })
+    })
+
+    it('http with redirection', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      waterfall([
+        (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, cb)
+        },
+        (serverA, cb) => {
+          const url = `http://127.0.0.1:${serverA.address().port}`
+          const handler = (req, res) => {
+            res.statusCode = 302
+            res.setHeader('Location', url)
+            res.end()
+          }
+          startTestServer(handler, (err, serverB) => {
+            if (err) return cb(err)
+            cb(null, { a: serverA, b: serverB })
+          })
+        }
+      ], (err, servers) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
-        done()
+
+        ipfs.add(data, (err, res) => {
+          expect(err).to.not.exist()
+
+          const expectedHash = res[0].hash
+          const url = `http://127.0.0.1:${servers.b.address().port}`
+
+          ipfs.util.addFromURL(url, (err, result) => {
+            expect(err).to.not.exist()
+            expect(result[0].hash).to.equal(expectedHash)
+            done()
+          })
+        })
       })
     })
 
-    it('with only-hash=true', function () {
-      this.timeout(40 * 1000)
+    it('https with redirection', (done) => {
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      waterfall([
+        (cb) => {
+          const handler = (req, res) => {
+            res.write(data)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, cb)
+        },
+        (serverA, cb) => {
+          const url = `https://127.0.0.1:${serverA.address().port}`
+          const handler = (req, res) => {
+            res.statusCode = 302
+            res.setHeader('Location', url)
+            res.end()
+          }
+          startTestServer(handler, { secure: true }, (err, serverB) => {
+            if (err) return cb(err)
+            cb(null, { a: serverA, b: serverB })
+          })
+        }
+      ], (err, servers) => {
+        expect(err).to.not.exist()
+
+        ipfs.add(data, (err, res) => {
+          expect(err).to.not.exist()
+
+          const expectedHash = res[0].hash
+          const url = `https://127.0.0.1:${servers.b.address().port}`
 
-      return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true })
-        .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000))
+          ipfs.util.addFromURL(url, (err, result) => {
+            expect(err).to.not.exist()
+            expect(result[0].hash).to.equal(expectedHash)
+            done()
+          })
+        })
+      })
     })
 
-    it('with wrap-with-directory=true', function (done) {
-      this.timeout(40 * 1000)
+    it('with only-hash=true', (done) => {
+      const handler = (req, res) => {
+        res.write(`TEST${Date.now()}`)
+        res.end()
+      }
 
-      ipfs.util.addFromURL('http://ipfs.io/ipfs/QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61/969165.jpg?foo=bar#buzz', {
-        wrapWithDirectory: true
-      }, (err, result) => {
+      startTestServer(handler, (err, server) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
-        expect(result[0].path).to.equal('969165.jpg')
-        expect(result[1].hash).to.equal('QmWjppACLcFLQ2qL38unKQvJBhXH3RUtcGLPk7zmrTwV61')
-        expect(result.length).to.equal(2)
-        done()
+
+        const url = `http://127.0.0.1:${server.address().port}/`
+
+        ipfs.util.addFromURL(url, { onlyHash: true }, (err, res) => {
+          expect(err).to.not.exist()
+
+          // A successful object.get for this size data took my laptop ~14ms
+          let didTimeout = false
+          const timeoutId = setTimeout(() => {
+            didTimeout = true
+            done()
+          }, 500)
+
+          ipfs.object.get(res[0].hash, () => {
+            clearTimeout(timeoutId)
+            if (didTimeout) return
+            expect(new Error('did not timeout')).to.not.exist()
+          })
+        })
       })
     })
 
-    it('with wrap-with-directory=true and URL-escaped file name', function (done) {
-      this.timeout(40 * 1000)
+    it('with wrap-with-directory=true', (done) => {
+      const filename = `TEST${Date.now()}.txt`
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => startTestServer((req, res) => {
+          res.write(data)
+          res.end()
+        }, cb),
+        expectedResult: (cb) => {
+          ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb)
+        }
+      }, (err, taskResult) => {
+        expect(err).to.not.exist()
+
+        const { server, expectedResult } = taskResult
+        const url = `http://127.0.0.1:${server.address().port}/${filename}?foo=bar#buzz`
+
+        ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
+      })
+    })
 
-      // Sample URL contains URL-escaped ( ) and local diacritics
-      ipfs.util.addFromURL('https://upload.wikimedia.org/wikipedia/commons/thumb/c/cf/Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg/320px-Doma%C5%BElice%2C_Jir%C3%A1skova_43_%289102%29.jpg?foo=bar#buzz', {
-        wrapWithDirectory: true
-      }, (err, result) => {
+    it('with wrap-with-directory=true and URL-escaped file name', (done) => {
+      const filename = '320px-Domažlice,_Jiráskova_43_(9102).jpg'
+      const data = Buffer.from(`TEST${Date.now()}`)
+
+      parallel({
+        server: (cb) => startTestServer((req, res) => {
+          res.write(data)
+          res.end()
+        }, cb),
+        expectedResult: (cb) => {
+          ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb)
+        }
+      }, (err, taskResult) => {
         expect(err).to.not.exist()
-        expect(result[0].hash).to.equal('QmRJ9ExxSMV4BLF9ZJUb2mLngupm6BXZEek755VHGTJo2Y')
-        expect(result[0].path).to.equal('320px-Domažlice,_Jiráskova_43_(9102).jpg')
-        expect(result[1].hash).to.equal('QmbxsHFU3sCfr8wszDHuDLA76C2xCv9HT8L3aC1pBwgaHk')
-        expect(result.length).to.equal(2)
-        done()
+
+        const { server, expectedResult } = taskResult
+        const url = `http://127.0.0.1:${server.address().port}/${encodeURIComponent(filename)}?foo=bar#buzz`
+
+        ipfs.util.addFromURL(url, { wrapWithDirectory: true }, (err, result) => {
+          expect(err).to.not.exist()
+          expect(result).to.deep.equal(expectedResult)
+          done()
+        })
       })
     })