Skip to content

Commit

Permalink
Merge pull request #123 from ipfs/datastore
Browse files Browse the repository at this point in the history 
feat: migrate to datastore from pull-blob-store
  • Loading branch information
@daviddias
daviddias authored Mar 21, 2017
2 parents 64f3062 + 5872e31 commit 86af686
Show file tree
Hide file tree
Showing 49 changed files with 1,048 additions and 813 deletions.
172 changes: 62 additions & 110 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,13 @@ This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs
## Table of Contents

- [Background](#background)
- [Good to know (historical context)](#good-to-know-historical-context)
- [Install](#install)
- [npm](#npm)
- [Use in Node.js](#use-in-nodejs)
- [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler)
- [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag)
- [Usage](#usage)
- [API](#api)
- [var repo = new IPFSRepo(path, opts)](#var-repo--new-ipfsrepopath-opts)
- [repo.exists(cb)](#repoexistscb)
- [repo.version.get(cb(err, version))](#repoversiongetcberr-version)
- [repo.version.set(version, cb(err))](#repoversionsetversion-cberr)
- [repo.config.get(cb(err, config))](#repoconfiggetcberr-config)
- [repo.config.set(config, cb(err))](#repoconfigsetconfig-cberr)
- [repo.keys](#repokeys)
- [repo.blockstore.putStream()](#)
- [repo.blockstore.getStream(key, extension)](#)
- [repo.datastore](#repodatastore)
- [Contribute](#contribute)
- [License](#license)

Expand All @@ -45,62 +34,60 @@ This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs
Here is the architectural reasoning for this repo:

```bash
┌─────────────────────────────────┐
│ interface defined by Repo Spec │
├─────────────────────────────────┤
│ │ ┌──────────────────────┐
│ │ │ interface-pull-blob-store │
│ IPFS REPO │─────────────────────────────────▶│ interface │
│ │ ├──────────────────────┤
│ │ │ locks │
└─────────────────────────────────┘ └──────────────────────┘
│
┌──────────┴────┬───────────────┬───────────────┬───────────────┬───────────────┐
▼ ▼ ▼ ▼ ▼ ▼
┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐
│ abstract │ │ abstract │ │ abstract │ │ abstract │ │ abstract │ │ abstract │
│ -blob │ │ -blob │ │ -blob │ │ -blob │ │ -blob │ │ -blob │
│ -store │ │ -store │ │ -store │ │ -store │ │ -store │ │ -store │
│ interface │ │ interface │ │ interface │ │ interface │ │ interface │ │ interface │
├───────────┤ ├───────────┤ ├───────────┤ ├───────────┤ ├───────────┤ ├───────────┤
│ │ │ │ │ │ │ │ │ │ │ │
│ keys │ │ config │ │ blockstore │ │ datastore │ │ logs │ │ version │
│ │ │ │ │ │ │ │ │ │ │ │
└───────────┘ └───────────┘ └───────────┘ └───────────┘ └───────────┘ └───────────┘
┌────────────────────────────────────────┐
│ IPFSRepo │
└────────────────────────────────────────┘
┌─────────────────┐
│ / │
├─────────────────┤
│ Datastore │
└─────────────────┘
│
┌───────────┴───────────┐
│ │
┌─────────────────┐ ┌─────────────────┐
│ /blocks │ │ /datastore │
├─────────────────┤ ├─────────────────┤
│ Datastore │ │ LevelDatastore │
└─────────────────┘ └─────────────────┘

┌────────────────────────────────────────┐ ┌────────────────────────────────────────┐
│ IPFSRepo - Default Node.js │ │ IPFSRepo - Default Browser │
└────────────────────────────────────────┘ └────────────────────────────────────────┘
┌─────────────────┐ ┌─────────────────┐
│ / │ │ / │
├─────────────────┤ ├─────────────────┤
│ FsDatastore │ │LevelJSDatastore │
└─────────────────┘ └─────────────────┘
│ │
┌───────────┴───────────┐ ┌───────────┴───────────┐
│ │ │ │
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ /blocks │ │ /datastore │ │ /blocks │ │ /datastore │
├─────────────────┤ ├─────────────────┤ ├─────────────────┤ ├─────────────────┤
│ FlatfsDatastore │ │LevelDBDatastore │ │LevelJSDatastore │ │LevelJSDatastore │
└─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘
```

This provides a well defined interface for creating and interacting with an IPFS
Repo backed by a group of abstract backends for keys, configuration, logs, and
more. Each of the individual repos has an interface defined by
[interface-pull-blob-store](https://github.com/ipfs/interface-pull-blob-store): this
enables us to make IPFS Repo portable (running on Node.js vs the browser) and
accept different types of storage mechanisms for each repo (fs, levelDB, etc).

### Good to know (historical context)

- The datastore folder holds the legacy version of datastore, still built in levelDB, there is a current endeavour of pushing it to fs completely.
- The blocks folder is the current version of datastore.
- The keys repo doesn't exist yet, as the private key is simply stored inside config
This provides a well defined interface for creating and interacting with an IPFS repo.

## Install

### npm

```sh
> npm i ipfs-repo
> npm install ipfs-repo
```

### Use in Node.js

```JavaScript
```js
var IPFSRepo = require('ipfs-repo')
```

### Use in a browser with browserify, webpack or any other bundler

The code published to npm that gets loaded on require is in fact a ES5 transpiled version with the right shims added. This means that you can require it and use with your favourite bundler without having to adjust asset management process.

```JavaScript
```js
var IPFSRepo = require('ipfs-repo')
```

Expand All @@ -119,77 +106,42 @@ Loading this module through a script tag will make the `IpfsRepo` obj available
Example:

```js
var inMemoryBS = require('interface-pull-blob-store')
// inMemoryBS is an "in memory" blob store, you can find others at:
// https://github.com/ipfs/interface-pull-blob-store#modules-that-use-this
const Repo = require('ipfs-repo')
const repo = new Repo('/tmp/ipfs-repo')

var IPFSRepo = require('ipfs-repo')
var repo = new IPFSRepo('/Users/someone/.ipfs', {
stores: inMemoryBS
})
```
repo.init({ cool: 'config' }, (err) => {
if (err) {
throw err
}

## API
repo.open((err) => {
if (err) {
throw err
}

```js
var IPFSRepo = require('ipfs-repo')
console.log('repo is ready')
})
})
```

### var repo = new IPFSRepo(path, opts)

Creates a **reference** to an IPFS repository at the path `path`. This does
*not* create the repo, but is an object that refers to the repo at such a path.

Valid keys for `opts` include:
This now has created the following structure, either on disk or as an in memory representation:

- `stores`: either an
[interface-pull-blob-store](https://github.com/ipfs/interface-pull-blob-store), or a
map of the form

```js
{
keys: someBlobStore,
config: someBlobStore,
datastore: someBlobStore,
logs: someBlobStore,
locks: someBlobStore,
version: someBlobStore
}
```
├── blocks
│   ├── SHARDING
│ └── _README
├── config
├── datastore
└── version
```

If you use the former form, all of the sub-blob-stores will use the same store.

### repo.exists(cb)

Check if the repo you are going to access already exists. Calls the callback
`cb(err, exists)`, where `exists` is true or false.

### repo.version.get(cb(err, version))
### repo.version.set(version, cb(err))

Read/write the version number of the repository. The version number is the repo version number.

### repo.config.get(cb(err, config))
### repo.config.set(config, cb(err))

Read/write the configuration object of the repository.

### repo.keys

Read/write keys inside the repo. This feature will be expanded once
[IPRS](https://github.com/ipfs/specs/tree/master/records) and
[KeyChain](https://github.com/ipfs/specs/tree/master/keychain) are finalized and implemented on go-ipfs.

### repo.blockstore.putStream()
### repo.datastore.getStream(key, extension)
### repo.datastore.has(key, extension, cb)
### repo.datastore.delete(key, extension, cb)
## API

Read and write buffers to/from the repo's block store.
See https://ipfs.github.io/js-ipfs-repo

### repo.datastore
## Notes

**WIP**
- [Explanation of how repo is structured](https://github.com/ipfs/js-ipfs-repo/pull/111#issuecomment-279948247)

## Contribute

Expand Down
18 changes: 18 additions & 0 deletions example.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
'use strict'

const Repo = require('ipfs-repo')
const repo = new Repo('/Users/awesome/.jsipfs')

repo.init({my: 'config'}, (err) => {
if (err) {
throw err
}

repo.open((err) => {
if (err) {
throw err
}

console.log('repo is ready')
})
})
53 changes: 32 additions & 21 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,55 +3,66 @@
"version": "0.11.3",
"description": "IPFS Repo implementation",
"main": "src/index.js",
"browser": {
"./src/lock.js": "./src/lock-memory.js",
"./src/default-options.js": "./src/default-options-browser.js"
},
"scripts": {
"test": "aegir-test",
"test:node": "aegir-test node",
"test:browser": "aegir-test browser",
"build": "aegir-build",
"lint": "aegir-lint",
"release": "aegir-release",
"release-minor": "aegir-release --type minor",
"release-major": "aegir-release --type major",
"release": "aegir-release --docs",
"release-minor": "aegir-release --type minor --docs",
"release-major": "aegir-release --type major --docs",
"coverage": "aegir-coverage",
"coverage-publish": "aegir-coverage publish"
"coverage-publish": "aegir-coverage publish",
"docs": "aegir-docs"
},
"repository": {
"type": "git",
"url": "https://github.com/ipfs/js-ipfs-repo.git"
},
"keywords": [
"IPFS"
"IPFS",
"libp2p",
"datastore"
],
"pre-commit": [
"lint",
"test"
],
"homepage": "https://github.com/ipfs/js-ipfs-repo",
"engines": {
"node": ">=4.0.0"
"node": ">=4.0.0",
"npm": ">=3.0.0"
},
"devDependencies": {
"aegir": "^10.0.0",
"buffer-loader": "^0.0.1",
"aegir": "^11.0.0",
"chai": "^3.5.0",
"fs-pull-blob-store": "~0.4.1",
"idb-pull-blob-store": "~0.5.1",
"interface-pull-blob-store": "~0.6.0",
"dirty-chai": "^1.2.2",
"lodash": "^4.17.4",
"multihashes": "~0.4.0",
"memdown": "^1.2.4",
"multihashes": "~0.4.4",
"ncp": "^2.0.0",
"pre-commit": "^1.2.2",
"rimraf": "^2.5.4"
"rimraf": "^2.6.1"
},
"dependencies": {
"async": "^2.1.4",
"async": "^2.1.5",
"base32.js": "^0.1.0",
"ipfs-block": "~0.5.5",
"lock": "^0.1.3",
"multihashes": "~0.3.3",
"pull-defer": "^0.2.2",
"pull-stream": "^3.5.0",
"pull-write": "^1.1.1",
"cids": "^0.4.2",
"datastore-core": "^0.1.0",
"datastore-fs": "^0.1.1",
"datastore-level": "^0.1.0",
"debug": "^2.6.3",
"interface-datastore": "^0.1.1",
"ipfs-block": "~0.6.0",
"level-js": "^2.2.4",
"leveldown": "^1.6.0",
"lock-me": "^1.0.2",
"multiaddr": "^2.2.2",
"safe-buffer": "^5.0.1"
},
"license": "MIT",
Expand All @@ -69,4 +80,4 @@
"nginnever <ginneversource@gmail.com>",
"npmcdn-to-unpkg-bot <npmcdn-to-unpkg-bot@users.noreply.github.com>"
]
}
}
Loading

0 comments on commit 86af686

Please sign in to comment.