Skip to content
This repository has been archived by the owner on Mar 10, 2020. It is now read-only.

Only hash #717

Merged
merged 2 commits into from
Mar 15, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
"hapi": "^17.2.2",
"interface-ipfs-core": "~0.55.1",
"ipfs": "~0.28.0",
"ipfsd-ctl": "~0.29.1",
"ipfsd-ctl": "~0.30.1",
"pre-commit": "^1.2.2",
"socket.io": "^2.0.4",
"socket.io-client": "^2.0.4",
Expand Down
2 changes: 1 addition & 1 deletion src/files/add-pull-stream.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ module.exports = (send) => {
return (options) => {
options = options || {}
options.converter = FileResultStreamConverter
return toPull(SendFilesStream(send, 'add')(options))
return toPull(SendFilesStream(send, 'add')({ qs: options }))
}
}
2 changes: 1 addition & 1 deletion src/files/add.js
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ module.exports = (send) => {

const files = [].concat(_files)

const stream = createAddStream(options)
const stream = createAddStream({ qs: options })
const concat = ConcatStream((result) => callback(null, result))
stream.once('error', callback)
stream.pipe(concat)
Expand Down
2 changes: 1 addition & 1 deletion src/files/write.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ module.exports = (send) => {
converter: FileResultStreamConverter
}

const stream = sendFilesStream(options)
const stream = sendFilesStream({ qs: options })
const concat = concatStream((result) => callback(null, result))
stream.once('error', callback)
stream.pipe(concat)
Expand Down
5 changes: 2 additions & 3 deletions src/util/fs-add.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@

const isNode = require('detect-node')
const promisify = require('promisify-es6')
const moduleConfig = require('../utils/module-config')
const SendOneFile = require('../utils/send-one-file-multiple-results')
const FileResultStreamConverter = require('../utils/file-result-stream-converter')

module.exports = (arg) => {
const sendOneFile = SendOneFile(moduleConfig(arg), 'add')
module.exports = (send) => {
const sendOneFile = SendOneFile(send, 'add')

return promisify((path, opts, callback) => {
if (typeof opts === 'function' &&
Expand Down
5 changes: 2 additions & 3 deletions src/util/url-add.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,11 @@
const promisify = require('promisify-es6')
const parseUrl = require('url').parse
const request = require('../utils/request')
const moduleConfig = require('../utils/module-config')
const SendOneFile = require('../utils/send-one-file-multiple-results')
const FileResultStreamConverter = require('../utils/file-result-stream-converter')

module.exports = (arg) => {
const sendOneFile = SendOneFile(moduleConfig(arg), 'add')
module.exports = (send) => {
const sendOneFile = SendOneFile(send, 'add')

return promisify((url, opts, callback) => {
if (typeof (opts) === 'function' &&
Expand Down
33 changes: 14 additions & 19 deletions src/utils/send-files-stream.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ module.exports = (send, path) => {
let ended = false
let writing = false

options = options || {}
options = options ? Object.assign({}, options, options.qs) : {}

const multipart = new Multipart()

Expand All @@ -42,7 +42,7 @@ module.exports = (send, path) => {
retStream._write = (file, enc, _next) => {
const next = once(_next)
try {
const files = prepareFile(file, Object.assign({}, options, options.qs))
const files = prepareFile(file, options)
.map((file) => Object.assign({headers: headers(file)}, file))

writing = true
Expand Down Expand Up @@ -75,23 +75,10 @@ module.exports = (send, path) => {

const qs = options.qs || {}

if (options['cid-version'] != null) {
qs['cid-version'] = options['cid-version']
} else if (options.cidVersion != null) {
qs['cid-version'] = options.cidVersion
}

if (options['raw-leaves'] != null) {
qs['raw-leaves'] = options['raw-leaves']
} else if (options.rawLeaves != null) {
qs['raw-leaves'] = options.rawLeaves
}

if (options.hash != null) {
qs.hash = options.hash
} else if (options.hashAlg != null) {
qs.hash = options.hashAlg
}
qs['cid-version'] = propOrProp(options, 'cid-version', 'cidVersion')
qs['raw-leaves'] = propOrProp(options, 'raw-leaves', 'rawLeaves')
qs['only-hash'] = propOrProp(options, 'only-hash', 'onlyHash')
qs.hash = propOrProp(options, 'hash', 'hashAlg')
Copy link

@AtkinsChang AtkinsChang Mar 23, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please don't directly assign qs which is a function parameters in API.
This will cause error if user pass read only object as option.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @AtkinsChang. Did this break something that used to work? We've been assigning to qs like this for a few months now so perhaps it is not just the assignment if you've used any of those versions.

which is a function parameters in API

I'm sorry, I don't get what you mean here. Can you give a clear example?

Copy link

@AtkinsChang AtkinsChang Mar 24, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JonKrone
For example, in src/files/add.js:voptions is passed directly to createAddStream({ qs: options }).
In src/utils/send-files-stream:
line 34 options = options ? Object.assign({}, options, options.qs) : {}
line 76 const qs = options.qs || {}
So basically the qs is the options user pass to the api.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because of line 34 like you mention, it looks like qs here is actually a copy of the options passed. I think that this avoids problems with a read-only options object. For example, the below doesn't cause any problems:

const obj = {}
Object.defineProperty(obj, 'color', {
  writable: false,
  value: 'red'
})
const options = Object.assign({}, obj)
options.color= 'rouge'
console.log(options.color) // prints 'rouge'

Did this break something that worked for you before?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes this is the currently workaround I use.
But I think the internal ipfs api should use a copy of user's option instead of modifying it directly. It is a good practice to avoid reassigning function arguments, which may cause user hard to debug in some situation, for example: shared options between different apis.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AtkinsChang ipfs-api internally does use a copy of the user's options because of line 34 and does not mutate the options you pass in. Given that, I'm not sure this is the source of the problem you were having.

If you can reproduce it in some code please let me know or submit a PR with your proposed changes.


const args = {
path: path,
Expand Down Expand Up @@ -158,3 +145,11 @@ module.exports = (send, path) => {
return retStream
}
}

function propOrProp (source, prop1, prop2) {
if (prop1 in source) {
return source[prop1]
} else if (prop2 in source) {
return source[prop2]
}
}
50 changes: 50 additions & 0 deletions test/files.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const CID = require('cids')

const IPFSApi = require('../src')
const f = require('./utils/factory')
const expectTimeout = require('./utils/expect-timeout')

const testfile = loadFixture('test/fixtures/testfile.txt')

Expand Down Expand Up @@ -102,6 +103,19 @@ describe('.files (the MFS API part)', function () {
})
})

it('files.add with only-hash=true', function () {
this.slow(10 * 1000)
const content = String(Math.random() + Date.now())

return ipfs.files.add(Buffer.from(content), { onlyHash: true })
.then(files => {
expect(files).to.have.length(1)

// 'ipfs.object.get(<hash>)' should timeout because content wasn't actually added
return expectTimeout(ipfs.object.get(files[0].hash), 4000)
})
})

it('files.add with options', (done) => {
ipfs.files.add(testfile, { pin: false }, (err, res) => {
expect(err).to.not.exist()
Expand All @@ -113,6 +127,42 @@ describe('.files (the MFS API part)', function () {
})
})

it('files.add pins by default', (done) => {
const newContent = Buffer.from(String(Math.random()))

ipfs.pin.ls((err, pins) => {
expect(err).to.not.exist()
const initialPinCount = pins.length
ipfs.files.add(newContent, (err, res) => {
expect(err).to.not.exist()

ipfs.pin.ls((err, pins) => {
expect(err).to.not.exist()
expect(pins.length).to.eql(initialPinCount + 1)
done()
})
})
})
})

it('files.add with pin=false', (done) => {
const newContent = Buffer.from(String(Math.random()))

ipfs.pin.ls((err, pins) => {
expect(err).to.not.exist()
const initialPinCount = pins.length
ipfs.files.add(newContent, { pin: false }, (err, res) => {
expect(err).to.not.exist()

ipfs.pin.ls((err, pins) => {
expect(err).to.not.exist()
expect(pins.length).to.eql(initialPinCount)
done()
})
})
})
})

HASH_ALGS.forEach((name) => {
it(`files.add with hash=${name} and raw-leaves=false`, (done) => {
const content = String(Math.random() + Date.now())
Expand Down
73 changes: 55 additions & 18 deletions test/util.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ chai.use(dirtyChai)
const isNode = require('detect-node')
const path = require('path')
const fs = require('fs')
const os = require('os')

const IPFSApi = require('../src')
const f = require('./utils/factory')
const expectTimeout = require('./utils/expect-timeout')

describe('.util', () => {
if (!isNode) { return }
Expand Down Expand Up @@ -92,31 +94,66 @@ describe('.util', () => {
done()
})
})

it('with only-hash=true', function () {
this.slow(10 * 1000)
const content = String(Math.random() + Date.now())
const filepath = path.join(os.tmpdir(), `${content}.txt`)
fs.writeFileSync(filepath, content)

return ipfs.util.addFromFs(filepath, { onlyHash: true })
.then(out => {
fs.unlinkSync(filepath)
return expectTimeout(ipfs.object.get(out[0].hash), 4000)
})
})
})

it('.urlAdd http', function (done) {
this.timeout(20 * 1000)
describe('.urlAdd', () => {
it('http', function (done) {
this.timeout(20 * 1000)

ipfs.util.addFromURL('http://example.com/', (err, result) => {
expect(err).to.not.exist()
expect(result.length).to.equal(1)
done()
ipfs.util.addFromURL('http://example.com/', (err, result) => {
expect(err).to.not.exist()
expect(result.length).to.equal(1)
done()
})
})
})

it('.urlAdd https', (done) => {
ipfs.util.addFromURL('https://example.com/', (err, result) => {
expect(err).to.not.exist()
expect(result.length).to.equal(1)
done()
it('https', function (done) {
this.timeout(20 * 1000)

ipfs.util.addFromURL('https://example.com/', (err, result) => {
expect(err).to.not.exist()
expect(result.length).to.equal(1)
done()
})
})
})

it('.urlAdd http with redirection', (done) => {
ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
expect(err).to.not.exist()
expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
done()
it('http with redirection', function (done) {
this.timeout(20 * 1000)

ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => {
expect(err).to.not.exist()
expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
done()
})
})

it('.urlAdd http with redirection', (done) => {
ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
expect(err).to.not.exist()
expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
done()
})
})

it('with only-hash=true', function () {
this.timeout(10 * 1000)
this.slow(10 * 1000)

return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true })
.then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000))
})
})
})
16 changes: 16 additions & 0 deletions test/utils/expect-timeout.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
'use strict'

/**
* Resolve if @param promise hangs for at least @param ms, throw otherwise
* @param {Promise} promise promise that you expect to hang
* @param {Number} ms millis to wait
* @return {Promise}
*/
module.exports = (promise, ms) => {
return Promise.race([
promise.then((out) => {
throw new Error('Expected Promise to timeout but it was successful.')
}),
new Promise((resolve, reject) => setTimeout(resolve, ms))
])
}