diff --git a/lib/blockstore/abstract.js b/lib/blockstore/abstract.js new file mode 100644 index 0000000000..df7172aed9 --- /dev/null +++ b/lib/blockstore/abstract.js @@ -0,0 +1,100 @@ +/*! + * blockstore/abstract.js - abstract block store for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const Logger = require('blgr'); + +/** + * Abstract Block Store + * + * @alias module:blockstore.AbstractBlockStore + * @abstract + */ + +class AbstractBlockStore { + /** + * Create an abstract blockstore. + * @constructor + */ + + constructor(options) { + this.options = options || {}; + + if (this.options.logger != null) + this.logger = this.options.logger.context('blockstore'); + else + this.logger = Logger.global.context('blockstore'); + } + + /** + * This method opens any necessary resources and + * initializes the store to be ready to be queried. + * @returns {Promise} + */ + + async open() { + throw new Error('Abstract method.'); + } + + /** + * This method closes resources and prepares + * store to be closed. + * @returns {Promise} + */ + + async close() { + throw new Error('Abstract method.'); + } + + /** + * This method stores block data. The action should be idempotent. + * If the data is already stored, the behavior will be the same. Any + * concurrent requests to store the same data will produce the same + * result, and will not conflict with each other. + * @returns {Promise} + */ + + async write(hash, data) { + throw new Error('Abstract method.'); + } + + /** + * This method will retrieve block data. Smaller portions of + * the block can be read by using the offset and size arguments. + * @returns {Promise} + */ + + async read(hash, offset, size) { + throw new Error('Abstract method.'); + } + + /** + * This will free resources for storing the block data. This + * may not mean that the block is deleted, but that it should + * no longer consume any local storage resources. + * @returns {Promise} + */ + + async prune(hash) { + throw new Error('Abstract method.'); + } + + /** + * This will check if a block has been stored and is available. + * @returns {Promise} + */ + + async has(hash) { + throw new Error('Abstract method.'); + } +} + +/* + * Expose + */ + +module.exports = AbstractBlockStore; diff --git a/lib/blockstore/file.js b/lib/blockstore/file.js new file mode 100644 index 0000000000..5fa3af475d --- /dev/null +++ b/lib/blockstore/file.js @@ -0,0 +1,367 @@ +/*! + * blockstore/file.js - file block store for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const {isAbsolute, resolve, join} = require('path'); +const bdb = require('bdb'); +const assert = require('bsert'); +const fs = require('bfile'); +const bio = require('bufio'); +const Network = require('../protocol/network'); +const Block = require('../primitives/block'); +const AbstractBlockStore = require('./abstract'); +const {BlockRecord, FileRecord} = require('./records'); +const layout = require('./layout'); + +/** + * File Block Store + * + * @alias module:blockstore:FileBlockStore + * @abstract + */ + +class FileBlockStore extends AbstractBlockStore { + /** + * Create a blockstore that stores blocks in files. + * @constructor + */ + + constructor(options) { + super(); + + assert(isAbsolute(options.location), 'Location not absolute.'); + + this.location = options.location; + this.db = bdb.create({ + location: resolve(this.location, './index') + }); + this.maxFileLength = options.maxFileLength || 128 * 1024 * 1024; + + this.network = Network.primary; + + if (options.network != null) + this.network = Network.get(options.network); + } + + /** + * Compares the number of files in the directory + * with the recorded number of files. If there are any + * inconsistencies it will reindex all blocks. + * @private + * @returns {Promise} + */ + + async index() { + const regexp = /^blk(\d{5})\.dat$/; + const all = await fs.readdir(this.location); + const dats = all.filter(f => regexp.test(f)); + const filenos = dats.map(f => parseInt(f.match(regexp)[1])); + + let missing = false; + + for (const fileno of filenos) { + const rec = await this.db.get(layout.f.encode(fileno)); + if (!rec) { + missing = true; + break; + } + } + + if (!missing) + return; + + this.logger.info('Indexing FileBlockStore...'); + + for (const fileno of filenos) { + const b = this.db.batch(); + const filepath = this.filepath(fileno); + const data = await fs.readFile(filepath); + const reader = bio.read(data); + let magic = null; + let blocks = 0; + + while (reader.left() >= 4) { + magic = reader.readU32(); + if (magic !== this.network.magic) { + reader.seek(4); + continue; + } + + const length = reader.readU32(); + const position = reader.offset; + + const block = Block.fromReader(reader); + const hash = block.hash(); + + const blockrecord = new BlockRecord({ + file: fileno, + position: position, + length: length + }); + + blocks += 1; + b.put(layout.b.encode(hash), blockrecord.toRaw()); + } + + const filerecord = new FileRecord({ + blocks: blocks, + used: reader.offset, + length: this.maxFileLength + }); + + b.put(layout.f.encode(fileno), filerecord.toRaw()); + + await b.write(); + + this.logger.info(`Indexed ${blocks} blocks from ${filepath}...`); + } + } + + /** + * Opens the file block store. It will regenerate necessary block + * indexing if the index is missing or inconsistent. + * @returns {Promise} + */ + + async open() { + this.logger.info('Opening FileBlockStore...'); + + await this.db.open(); + await this.db.verify(layout.V.encode(), 'fileblockstore', 0); + + await this.index(); + } + + /** + * This closes the file block store and underlying + * databases for indexing. + */ + + async close() { + this.logger.info('Closing FileBlockStore...'); + + await this.db.close(); + } + + /** + * This method will determine the file path based on the file number + * and the current block data location. + * @param {Number} fileno - The number of the file. + * @returns {Promise} + */ + + filepath(fileno) { + const pad = 5; + + let num = fileno.toString(10); + + if (num.length > pad) + throw new Error('File number too large.'); + + while (num.length < pad) + num = `0${num}`; + + return join(this.location, `blk${num}.dat`); + } + + /** + * This method will select and potentially allocate a file to + * write a block based on the size. + * @param {Number} length - The number of bytes of the data to be written. + * @returns {Promise} + */ + + async allocate(length) { + if (length > this.maxFileLength) + throw new Error('Block length above max file length.'); + + let fileno = 0; + let filerecord = null; + let filepath = null; + + const last = await this.db.get(layout.R.encode()); + if (last) + fileno = bio.read(last).readU32(); + + filepath = this.filepath(fileno); + + const rec = await this.db.get(layout.f.encode(fileno)); + + if (rec) { + filerecord = FileRecord.fromRaw(rec); + } else { + filerecord = new FileRecord({ + blocks: 0, + used: 0, + length: this.maxFileLength + }); + } + + if (filerecord.used + length > filerecord.length) { + fileno += 1; + filepath = this.filepath(fileno); + filerecord = new FileRecord({ + blocks: 0, + used: 0, + length: this.maxFileLength + }); + } + + return {fileno, filerecord, filepath}; + } + + /** + * This method stores block data in files. + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async write(hash, data) { + const mlength = 8; + const blength = data.length; + const length = data.length + mlength; + + const { + fileno, + filerecord, + filepath + } = await this.allocate(length); + + const mposition = filerecord.used; + const bposition = filerecord.used + mlength; + + const bwm = bio.write(mlength); + bwm.writeU32(this.network.magic); + bwm.writeU32(blength); + const magic = bwm.render(); + + const fd = await fs.open(filepath, 'a'); + + const mbytes = await fs.write(fd, magic, 0, mlength, mposition); + const bbytes = await fs.write(fd, data, 0, blength, bposition); + + await fs.close(fd); + + if (mbytes !== 8) + throw new Error('Could not write block magic.'); + + if (bbytes !== blength) + throw new Error('Could not write block.'); + + filerecord.blocks += 1; + filerecord.used += length; + + const b = this.db.batch(); + + const blockrecord = new BlockRecord({ + file: fileno, + position: bposition, + length: blength + }); + + b.put(layout.b.encode(hash), blockrecord.toRaw()); + b.put(layout.f.encode(fileno), filerecord.toRaw()); + + const bw = bio.write(4); + b.put(layout.R.encode(), bw.writeU32(fileno).render()); + + await b.write(); + } + + /** + * This method will retrieve block data. Smaller portions of the + * block (e.g. transactions) can be read by using the offset and + * length arguments. + * @param {Buffer} hash - The block hash + * @param {Number} offset - The offset within the block + * @param {Number} length - The number of bytes of the data + * @returns {Promise} + */ + + async read(hash, offset, length) { + const raw = await this.db.get(layout.b.encode(hash)); + if (!raw) + return null; + + const blockrecord = BlockRecord.fromRaw(raw); + + const filepath = this.filepath(blockrecord.file); + + let position = blockrecord.position; + + if (offset) + position += offset; + + if (!length) + length = blockrecord.length; + + if (offset + length > blockrecord.length) + throw new Error('Out-of-bounds read.'); + + const data = Buffer.alloc(length); + + const fd = await fs.open(filepath, 'r'); + await fs.read(fd, data, 0, length, position); + await fs.close(fd); + + return data; + } + + /** + * This will free resources for storing the block data. The block + * data may not be deleted from disk immediately, the index for + * the block is removed and will not be able to be read. The underlying + * file is unlinked when all blocks in a file have been pruned. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async prune(hash) { + const braw = await this.db.get(layout.b.encode(hash)); + if (!braw) + return false; + + const blockrecord = BlockRecord.fromRaw(braw); + + const fraw = await this.db.get(layout.f.encode(blockrecord.file)); + if (!fraw) + return false; + + const filerecord = FileRecord.fromRaw(fraw); + + filerecord.blocks -= 1; + + const b = this.db.batch(); + b.put(layout.f.encode(blockrecord.file), filerecord.toRaw()); + b.del(layout.b.encode(hash)); + await b.write(); + + if (filerecord.blocks === 0) { + const filepath = this.filepath(blockrecord.file); + await fs.unlink(filepath); + } + + return true; + } + + /** + * This will check if a block has been stored and is available. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async has(hash) { + return await this.db.has(layout.b.encode(hash)); + } +} + +/* + * Expose + */ + +module.exports = FileBlockStore; diff --git a/lib/blockstore/index.js b/lib/blockstore/index.js new file mode 100644 index 0000000000..77bf971543 --- /dev/null +++ b/lib/blockstore/index.js @@ -0,0 +1,15 @@ +/*! + * blockstore/index.js - bitcoin blockstore for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +/** + * @module blockstore + */ + +exports.AbstractBlockStore = require('./abstract'); +exports.FileBlockStore = require('./file'); +exports.LevelBlockStore = require('./level'); diff --git a/lib/blockstore/layout.js b/lib/blockstore/layout.js new file mode 100644 index 0000000000..5b4d6e39eb --- /dev/null +++ b/lib/blockstore/layout.js @@ -0,0 +1,30 @@ +/*! + * blockstore/layout.js - file block store data layout for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const bdb = require('bdb'); + +/* + * Database Layout: + * V -> db version + * R -> last file entry + * f[uint32] -> file entry + * b[hash] -> block entry + */ + +const layout = { + V: bdb.key('V'), + R: bdb.key('R'), + f: bdb.key('f', ['uint32']), + b: bdb.key('b', ['hash256']) +}; + +/* + * Expose + */ + +module.exports = layout; diff --git a/lib/blockstore/level.js b/lib/blockstore/level.js new file mode 100644 index 0000000000..42764c625c --- /dev/null +++ b/lib/blockstore/level.js @@ -0,0 +1,128 @@ +/*! + * blockstore/level.js - leveldb block store for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const {isAbsolute, resolve} = require('path'); +const bdb = require('bdb'); +const assert = require('bsert'); +const AbstractBlockStore = require('./abstract'); +const layout = require('./layout'); + +/** + * LevelDB Block Store + * + * @alias module:blockstore:LevelBlockStore + * @abstract + */ + +class LevelBlockStore extends AbstractBlockStore { + /** + * Create a blockstore that stores blocks in LevelDB. + * @constructor + */ + + constructor(options) { + super(); + + assert(isAbsolute(options.location), 'Location not absolute.'); + + this.location = options.location; + this.db = bdb.create({ + location: resolve(this.location, './index') + }); + } + + /** + * Opens the block storage. + * @returns {Promise} + */ + + async open() { + this.logger.info('Opening LevelBlockStore...'); + + await this.db.open(); + await this.db.verify(layout.V.encode(), 'levelblockstore', 0); + } + + /** + * Closes the block storage. + */ + + async close() { + this.logger.info('Closing LevelBlockStore...'); + + await this.db.close(); + } + + /** + * This method stores block data in LevelDB. + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async write(hash, data) { + this.db.put(layout.b.encode(hash), data); + } + + /** + * This method will retrieve block data. Smaller portions of the + * block (e.g. transactions) can be returned using the offset and + * length arguments. However, the entire block will be read as the + * data is stored in a key/value database. + * @param {Buffer} hash - The block hash + * @param {Number} offset - The offset within the block + * @param {Number} length - The number of bytes of the data + * @returns {Promise} + */ + + async read(hash, offset, length) { + let raw = await this.db.get(layout.b.encode(hash)); + + if (offset) { + if (offset + length > raw.length) + throw new Error('Out-of-bounds read.'); + + raw = raw.slice(offset, offset + length); + } + + return raw; + } + + /** + * This will free resources for storing the block data. The block + * data may not be immediately removed from disk, and will be reclaimed + * during LevelDB compaction. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async prune(hash) { + if (!await this.has(hash)) + return false; + + await this.db.del(layout.b.encode(hash)); + + return true; + } + + /** + * This will check if a block has been stored and is available. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async has(hash) { + return this.db.has(layout.b.encode(hash)); + } +} + +/* + * Expose + */ + +module.exports = LevelBlockStore; diff --git a/lib/blockstore/records.js b/lib/blockstore/records.js new file mode 100644 index 0000000000..1f75ce0040 --- /dev/null +++ b/lib/blockstore/records.js @@ -0,0 +1,149 @@ +/*! + * blockstore/records.js - block store records + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('bsert'); +const bio = require('bufio'); + +/** + * @module blockstore/records + */ + +/** + * Block Record + */ + +class BlockRecord { + /** + * Create a block record. + * @constructor + */ + + constructor(options = {}) { + this.file = options.file || 0; + this.position = options.position || 0; + this.length = options.length || 0; + + assert((this.file >>> 0) === this.file); + assert((this.position >>> 0) === this.position); + assert((this.length >>> 0) === this.length); + } + + /** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.file = br.readU32(); + this.position = br.readU32(); + this.length = br.readU32(); + + return this; + } + + /** + * Instantiate block record from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {BlockRecord} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } + + /** + * Serialize the block record. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(12); + + bw.writeU32(this.file); + bw.writeU32(this.position); + bw.writeU32(this.length); + + return bw.render(); + } +} + +/** + * File Record + */ + +class FileRecord { + /** + * Create a chain state. + * @constructor + */ + + constructor(options = {}) { + this.blocks = options.blocks || 0; + this.used = options.used || 0; + this.length = options.length || 0; + + assert((this.blocks >>> 0) === this.blocks); + assert((this.used >>> 0) === this.used); + assert((this.length >>> 0) === this.length); + } + + /** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.blocks = br.readU32(); + this.used = br.readU32(); + this.length = br.readU32(); + + return this; + } + + /** + * Instantiate file record from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {ChainState} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } + + /** + * Serialize the file record. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(12); + + bw.writeU32(this.blocks); + bw.writeU32(this.used); + bw.writeU32(this.length); + + return bw.render(); + } +} + +/* + * Expose + */ + +exports.BlockRecord = BlockRecord; +exports.FileRecord = FileRecord; + +module.exports = exports; diff --git a/test/blockstore-test.js b/test/blockstore-test.js new file mode 100644 index 0000000000..8b6646cb1c --- /dev/null +++ b/test/blockstore-test.js @@ -0,0 +1,624 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const Logger = require('blgr'); +const assert = require('./util/assert'); +const common = require('./util/common'); +const {resolve} = require('path'); +const fs = require('bfile'); +const {rimraf} = require('./util/common'); +const {mkdirp} = require('bfile'); +const random = require('bcrypto/lib/random'); + +const vectors = [ + common.readBlock('block300025'), + common.readBlock('block426884'), + common.readBlock('block898352') +]; + +const { + AbstractBlockStore, + FileBlockStore, + LevelBlockStore +} = require('../lib/blockstore'); + +const { + BlockRecord, + FileRecord +} = require('../lib/blockstore/records'); + +describe('BlockStore', function() { + describe('Abstract', function() { + let logger = null; + + function context(ctx) { + return {info: () => ctx}; + } + + beforeEach(() => { + logger = Logger.global; + Logger.global = {context}; + }); + + afterEach(() => { + Logger.global = logger; + }); + + it('construct with custom logger', async () => { + const store = new AbstractBlockStore({logger: {context}}); + assert(store.logger); + assert(store.logger.info); + assert.equal(store.logger.info(), 'blockstore'); + }); + + it('construct with default logger', async () => { + const store = new AbstractBlockStore(); + assert(store.logger); + assert(store.logger.info); + assert.equal(store.logger.info(), 'blockstore'); + }); + + it('has unimplemented base methods', async () => { + const methods = ['open', 'close', 'write', 'read', + 'prune', 'has']; + + const store = new AbstractBlockStore(); + + for (const method of methods) { + assert(store[method]); + + let err = null; + try { + await store[method](); + } catch (e) { + err = e; + } + assert(err, `Expected unimplemented method ${method}.`); + assert.equal(err.message, 'Abstract method.'); + } + }); + }); + + describe('Records', function() { + describe('BlockRecord', function() { + function constructError(options) { + let err = null; + + try { + new BlockRecord({ + file: options.file, + position: options.position, + length: options.length + }); + } catch (e) { + err = e; + } + + assert(err); + } + + function toAndFromRaw(options) { + const rec1 = new BlockRecord(options); + assert.equal(rec1.file, options.file); + assert.equal(rec1.position, options.position); + assert.equal(rec1.length, options.length); + + const raw = rec1.toRaw(); + const rec2 = BlockRecord.fromRaw(raw); + assert.equal(rec2.file, options.file); + assert.equal(rec2.position, options.position); + assert.equal(rec2.length, options.length); + } + + it('construct with correct options', () => { + const rec = new BlockRecord({ + file: 12, + position: 23392, + length: 4194304 + }); + assert.equal(rec.file, 12); + assert.equal(rec.position, 23392); + assert.equal(rec.length, 4194304); + }); + + it('construct null record', () => { + const rec = new BlockRecord(); + assert.equal(rec.file, 0); + assert.equal(rec.position, 0); + assert.equal(rec.length, 0); + }); + + it('fail with signed number (file)', () => { + constructError({file: -1, position: 1, length: 1}); + }); + + it('fail with signed number (position)', () => { + constructError({file: 1, position: -1, length: 1}); + }); + + it('fail with signed number (length)', () => { + constructError({file: 1, position: 1, length: -1}); + }); + + it('fail with non-32-bit number (file)', () => { + constructError({file: Math.pow(2, 32), position: 1, length: 1}); + }); + + it('fail with non-32-bit number (position)', () => { + constructError({file: 1, position: Math.pow(2, 32), length: 1}); + }); + + it('fail with non-32-bit number (length)', () => { + constructError({file: 1, position: 1, length: Math.pow(2, 32)}); + }); + + it('construct with max 32-bit numbers', () => { + const max = Math.pow(2, 32) - 1; + + const rec = new BlockRecord({ + file: max, + position: max, + length: max + }); + + assert(rec); + assert.equal(rec.file, max); + assert.equal(rec.position, max); + assert.equal(rec.length, max); + }); + + it('serialize/deserialize file record (min)', () => { + toAndFromRaw({file: 0, position: 0, length: 0}); + }); + + it('serialize/deserialize file record', () => { + toAndFromRaw({file: 12, position: 23392, length: 4194304}); + }); + + it('serialize/deserialize file record (max)', () => { + const max = Math.pow(2, 32) - 1; + toAndFromRaw({file: max, position: max, length: max}); + }); + }); + + describe('FileRecord', function() { + function constructError(options) { + let err = null; + + try { + new FileRecord({ + blocks: options.blocks, + used: options.used, + length: options.length + }); + } catch (e) { + err = e; + } + + assert(err); + } + + function toAndFromRaw(options) { + const rec1 = new FileRecord(options); + assert.equal(rec1.blocks, options.blocks); + assert.equal(rec1.used, options.used); + assert.equal(rec1.length, options.length); + + const raw = rec1.toRaw(); + const rec2 = FileRecord.fromRaw(raw); + assert.equal(rec2.blocks, options.blocks); + assert.equal(rec2.used, options.used); + assert.equal(rec2.length, options.length); + } + + it('construct with correct options', () => { + const rec = new FileRecord({ + blocks: 1, + used: 4194304, + length: 20971520 + }); + assert.equal(rec.blocks, 1); + assert.equal(rec.used, 4194304); + assert.equal(rec.length, 20971520); + }); + + it('fail to with signed number (blocks)', () => { + constructError({blocks: -1, used: 1, length: 1}); + }); + + it('fail to with signed number (used)', () => { + constructError({blocks: 1, used: -1, length: 1}); + }); + + it('fail to with signed number (length)', () => { + constructError({blocks: 1, used: 1, length: -1}); + }); + + it('fail to with non-32-bit number (blocks)', () => { + constructError({blocks: Math.pow(2, 32), used: 1, length: 1}); + }); + + it('fail to with non-32-bit number (used)', () => { + constructError({blocks: 1, used: Math.pow(2, 32), length: 1}); + }); + + it('fail to with non-32-bit number (length)', () => { + constructError({blocks: 1, used: 1, length: Math.pow(2, 32)}); + }); + + it('serialize/deserialize block record (min)', () => { + toAndFromRaw({blocks: 0, used: 0, length: 0}); + }); + + it('serialize/deserialize block record', () => { + toAndFromRaw({blocks: 10, used: 4194304, length: 20971520}); + }); + + it('serialize/deserialize block record (max)', () => { + const max = Math.pow(2, 32) - 1; + toAndFromRaw({blocks: max, used: max, length: max}); + }); + }); + }); + + describe('FileBlockStore (Unit)', function() { + const location = '/tmp/.bcoin/blocks'; + let store = null; + + before(() => { + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + }); + + describe('allocate', function() { + it('will fail with length above file max', async () => { + let err = null; + try { + await store.allocate(1025); + } catch (e) { + err = e; + } + assert(err); + assert.equal(err.message, 'Block length above max file length.'); + }); + }); + + describe('filepath', function() { + it('will give correct path (0)', () => { + const filepath = store.filepath(0); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00000.dat'); + }); + + it('will give correct path (1)', () => { + const filepath = store.filepath(7); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00007.dat'); + }); + + it('will give correct path (2)', () => { + const filepath = store.filepath(23); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00023.dat'); + }); + + it('will give correct path (3)', () => { + const filepath = store.filepath(456); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00456.dat'); + }); + + it('will give correct path (4)', () => { + const filepath = store.filepath(8999); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk08999.dat'); + }); + + it('will give correct path (5)', () => { + const filepath = store.filepath(99999); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk99999.dat'); + }); + + it('will fail over max size', () => { + let err = null; + try { + store.filepath(100000); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'File number too large.'); + }); + }); + }); + + describe('FileBlockStore (Integration 1)', function() { + const location = '/tmp/bcoin-blockstore-test'; + let store = null; + + beforeEach(async () => { + await rimraf(location); + await mkdirp(location); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + + await store.open(); + }); + + afterEach(async () => { + await store.close(); + }); + + it('will write and read a block', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const block2 = await store.read(hash); + + assert.bufferEqual(block1, block2); + }); + + it('will read a block w/ offset and length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 15; + + const block2 = await store.read(hash, offset, size); + + assert.bufferEqual(block1.slice(offset, offset + size), block2); + }); + + it('will fail to read w/ out-of-bounds length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 50; + + let err = null; + try { + await store.read(hash, offset, size); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Out-of-bounds read.'); + }); + + it('will allocate new files', async () => { + const blocks = []; + + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + blocks.push({hash, block}); + await store.write(hash, block); + const block2 = await store.read(hash); + assert.bufferEqual(block2, block); + } + + const first = await fs.stat(store.filepath(0)); + const second = await fs.stat(store.filepath(1)); + const third = await fs.stat(store.filepath(2)); + assert.equal(first.size, 952); + assert.equal(second.size, 952); + assert.equal(third.size, 272); + + const len = first.size + second.size + third.size - (8 * 16); + assert.equal(len, 128 * 16); + + for (let i = 0; i < 16; i++) { + const expect = blocks[i]; + const block = await store.read(expect.hash); + assert.bufferEqual(block, expect.block); + } + }); + + it('will return null if block not found', async () => { + const hash = random.randomBytes(32); + const block = await store.read(hash); + assert.strictEqual(block, null); + }); + + it('will check if block exists (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + }); + + it('will check if block exists (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + const exists = await store.has(hash); + assert.strictEqual(exists, true); + }); + + it('will prune blocks', async () => { + const hashes = []; + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + hashes.push(hash); + await store.write(hash, block); + } + + const first = await fs.stat(store.filepath(0)); + const second = await fs.stat(store.filepath(1)); + const third = await fs.stat(store.filepath(2)); + + const len = first.size + second.size + third.size - (8 * 16); + assert.equal(len, 128 * 16); + + for (let i = 0; i < 16; i++) { + const pruned = await store.prune(hashes[i]); + assert.strictEqual(pruned, true); + } + + assert.equal(await fs.exists(store.filepath(0)), false); + assert.equal(await fs.exists(store.filepath(1)), false); + assert.equal(await fs.exists(store.filepath(2)), false); + }); + }); + + describe('FileBlockStore (Integration 2)', function() { + const location = '/tmp/bcoin-blockstore-test'; + let store = null; + + beforeEach(async () => { + await rimraf(location); + await mkdirp(location); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 * 1024 + }); + + await store.open(); + }); + + afterEach(async () => { + await store.close(); + }); + + it('will import from files (e.g. db corruption)', async () => { + const blocks = []; + + for (let i = 0; i < vectors.length; i++) { + const [block] = vectors[i].getBlock(); + const hash = block.hash(); + const raw = block.toRaw(); + + blocks.push({hash, block: raw}); + await store.write(hash, raw); + } + + await store.close(); + + await rimraf(resolve(location, './index')); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + + await store.open(); + + for (let i = 0; i < vectors.length; i++) { + const expect = blocks[i]; + const block = await store.read(expect.hash); + assert.equal(block.length, expect.block.length); + assert.bufferEqual(block, expect.block); + } + }); + }); + + describe('LevelBlockStore', function() { + const location = '/tmp/bcoin-blockstore-test'; + let store = null; + + beforeEach(async () => { + await rimraf(location); + await mkdirp(location); + + store = new LevelBlockStore({ + location: location + }); + + await store.open(); + }); + + afterEach(async () => { + await store.close(); + }); + + it('will write and read a block', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const block2 = await store.read(hash); + + assert.bufferEqual(block1, block2); + }); + + it('will read a block w/ offset and length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 15; + + const block2 = await store.read(hash, offset, size); + + assert.bufferEqual(block1.slice(offset, offset + size), block2); + }); + + it('will fail to read w/ out-of-bounds length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 50; + + let err = null; + try { + await store.read(hash, offset, size); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Out-of-bounds read.'); + }); + + it('will check if block exists (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + }); + + it('will check if block exists (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + const exists = await store.has(hash); + assert.strictEqual(exists, true); + }); + + it('will prune blocks (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + const pruned = await store.prune(hash); + assert.strictEqual(pruned, true); + const block2 = await store.read(hash); + assert.strictEqual(block2, null); + }); + + it('will prune blocks (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + const pruned = await store.prune(hash); + assert.strictEqual(pruned, false); + }); + }); +}); diff --git a/test/util/common.js b/test/util/common.js index 9928eb4509..bdaa86203d 100644 --- a/test/util/common.js +++ b/test/util/common.js @@ -85,6 +85,14 @@ common.writeTX = function writeTX(name, tx, view) { common.writeFile(`${name}-undo.raw`, undoRaw); }; +common.rimraf = async function(p) { + const allowed = new RegExp('^\/tmp\/(.*)$'); + if (!allowed.test(p)) + throw new Error(`Path not allowed: ${p}.`); + + return await fs.rimraf(p); +}; + function parseUndo(data) { const br = bio.read(data); const items = [];