Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

watch is a stream #121

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

watch is a stream #121

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c6690b4
watch is a stream
mafintosh Jun 4, 2023
7cc1ee8
docs
mafintosh Jun 4, 2023
9eeb69b
hwm should be 1
mafintosh Jun 4, 2023
3064bb9
add eager mode to watch
mafintosh Jun 4, 2023
26f5edb
getAndWatch is a stream also now and some refactors
mafintosh Jun 4, 2023
b0226ca
standard
mafintosh Jun 4, 2023
d15a046
make checkout better
mafintosh Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 9 additions & 18 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,21 +267,9 @@ and `right` will be null, and vice versa.
If the entries are causally equal (i.e. the have the same seq), they are not
returned, only the diff.

#### `const entryWatcher = await db.getAndWatch(key, [options])`
#### `const stream = db.watch([range], [options])`

Returns a watcher which listens to changes on the given key.

`entryWatcher.node` contains the current entry in the same format as the result of `bee.get(key)`, and will be updated as it changes.

By default, the node will have the bee's key- and value encoding, but you can overwrite it by setting the `keyEncoding` and `valueEncoding` options.

You can listen to `entryWatcher.on('update')` to be notified when the value of node has changed.

Call `await watcher.close()` to stop the watcher.

#### `const watcher = db.watch([range])`

Listens to changes that are on the optional `range`.
Watch for changes that are on the optional `range`.

`range` options are the same as `db.createReadStream` except for `reverse`.

Expand All @@ -301,13 +289,16 @@ Don't close those snapshots yourself because they're used internally, let them b

Watchers on subs and checkouts are not supported. Instead, use the range option to limit scope.

`await watcher.ready()`
`await watcher.opened`

Waits until the watcher is loaded and detecting changes, if you explicitly need that.

Waits until the watcher is loaded and detecting changes.
#### `const stream = db.getAndWatch(key, [options])`

`await watcher.close()`
Similar to above, except it only watches for a single key.

Stops the watcher. You could also stop it by using `break` in the loop.
Yields `[currentNode, previousNode]` objects when watch updates. If a node has been deleted,
it will be null in the array.

#### `const snapshot = db.checkout(version)`

Expand Down
282 changes: 11 additions & 271 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ const mutexify = require('mutexify/promise')
const b4a = require('b4a')
const safetyCatch = require('safety-catch')
const ReadyResource = require('ready-resource')
const debounce = require('debounceify')

const RangeIterator = require('./iterators/range')
const HistoryIterator = require('./iterators/history')
const DiffIterator = require('./iterators/diff')
const { RangeWatchStream, KeyWatchStream } = require('./lib/watch-stream')
const Extension = require('./lib/extension')
const { YoloIndex, Node, Header } = require('./lib/messages')

Expand Down Expand Up @@ -406,54 +406,41 @@ class Hyperbee extends ReadyResource {

watch (range, opts) {
if (!this._watchers) throw new Error('Can only watch the main bee instance')
return new Watcher(this, range, opts)
return new RangeWatchStream(this, range, opts)
}

async getAndWatch (key, opts) {
getAndWatch (key, opts) {
if (!this._watchers) throw new Error('Can only watch the main bee instance')

const watcher = new EntryWatcher(this, key, opts)
await watcher._debouncedUpdate()

if (this.closing) {
await watcher.close()
throw new Error('Bee closed')
}

return watcher
return new KeyWatchStream(this, key, opts)
}

_onappend () {
for (const watcher of this._watchers) {
watcher._onappend()
}

for (const watcher of this._entryWatchers) {
watcher._onappend()
}
}

_ontruncate () {
for (const watcher of this._watchers) {
watcher._ontruncate()
}

for (const watcher of this._entryWatchers) {
watcher._ontruncate()
}
}

_makeSnapshot () {
// TODO: better if we could encapsulate this in hypercore in the future
return this._checkout <= this.core.length ? this.core.snapshot() : this.core.session()
}

session (opts) {
return this.checkout(0, opts)
}

checkout (version, opts = {}) {
// same as above, just checkout isn't set yet...

const snap = opts.reuseSession
? this
: version <= this.core.length ? this.core.snapshot() : this.core.session()
: (version > 0 && version <= this.core.length) ? this.core.snapshot() : this.core.session()
HDegroote marked this conversation as resolved.
Show resolved Hide resolved

return new Hyperbee(snap, {
_view: true,
Expand Down Expand Up @@ -505,14 +492,8 @@ class Hyperbee extends ReadyResource {
this.core.off('append', this._onappendBound)
this.core.off('truncate', this._ontruncateBound)

while (this._watchers.length) {
await this._watchers[this._watchers.length - 1].close()
}
}

if (this._entryWatchers) {
while (this._entryWatchers.length) {
await this._entryWatchers[this._entryWatchers.length - 1].close()
for (let i = this._watchers.length - 1; i >= 0; i--) {
this._watchers[i].destroy()
}
}

Expand Down Expand Up @@ -941,243 +922,6 @@ class Batch {
}
}

class EntryWatcher extends ReadyResource {
constructor (bee, key, opts = {}) {
super()

this.keyEncoding = opts.keyEncoding || bee.keyEncoding
this.valueEncoding = opts.valueEncoding || bee.valueEncoding

this.index = bee._entryWatchers.push(this) - 1
this.bee = bee

this.key = key
this.node = null

this._forceUpdate = false
this._debouncedUpdate = debounce(this._processUpdate.bind(this))
}

_close () {
const top = this.bee._entryWatchers.pop()
if (top !== this) {
top.index = this.index
this.bee._entryWatchers[top.index] = top
}
}

_onappend () {
this._debouncedUpdate()
}

_ontruncate () {
this._forceUpdate = true
this._debouncedUpdate()
}

async _processUpdate () {
const force = this._forceUpdate
this._forceUpdate = false

let newNode
try {
newNode = await this.bee.get(this.key, {
keyEncoding: this.keyEncoding,
valueEncoding: this.valueEncoding
})
} catch (e) {
if (e.code === 'SNAPSHOT_NOT_AVAILABLE') {
// There was a truncate event before the get resolved
// So this handler will run again anyway
return
} else if (this.bee.closing) {
this.close().catch(safetyCatch)
return
}
this.emit('error', e)
return
}

if (force || newNode?.seq !== this.node?.seq) {
this.node = newNode
this.emit('update')
}
}
}

class Watcher extends ReadyResource {
constructor (bee, range, opts = {}) {
super()

this.keyEncoding = opts.keyEncoding || bee.keyEncoding
this.valueEncoding = opts.valueEncoding || bee.valueEncoding
this.index = bee._watchers.push(this) - 1
this.bee = bee
this.core = bee.core

this.latestDiff = 0
this.range = range
this.map = opts.map || defaultWatchMap

this.current = null
this.previous = null
this.currentMapped = null
this.previousMapped = null
this.stream = null

this._lock = mutexify()
this._flowing = false
this._resolveOnChange = null
this._differ = opts.differ || defaultDiffer

this.on('newListener', autoFlowOnUpdate)
this.ready().catch(safetyCatch)
}

async _consume () {
if (this._flowing) return
try {
for await (const _ of this) {} // eslint-disable-line
} catch {}
}

async _open () {
await this.bee.ready()

// Point from which to start watching
this.current = this.bee.snapshot({
keyEncoding: this.keyEncoding,
valueEncoding: this.valueEncoding
})
}

[Symbol.asyncIterator] () {
this._flowing = true
return this
}

_ontruncate () {
if (this.core.isAutobase) this._onappend()
}

_onappend () {
// TODO: this is a light hack / fix for non-sparse session reporting .length's inside batches
// the better solution is propably just to change non-sparse sessions to not report a fake length
if (!this.core.isAutobase && (!this.core.core || this.core.core.tree.length !== this.core.length)) return

const resolve = this._resolveOnChange
this._resolveOnChange = null
if (resolve) resolve()
}

async _waitForChanges () {
if (this.current.version < this.bee.version || this.closing) return

await new Promise(resolve => {
this._resolveOnChange = resolve
})
}

async next () {
try {
return await this._next()
} catch (err) {
if (this.closing) return { value: undefined, done: true }
await this.close()
throw err
}
}

async _next () {
const release = await this._lock()

try {
if (this.closing) return { value: undefined, done: true }

if (!this.opened) await this.ready()

while (true) {
await this._waitForChanges()

if (this.closing) return { value: undefined, done: true }

await this._closePrevious()
this.previous = this.current.snapshot()

await this._closeCurrent()
this.current = this.bee.snapshot({
keyEncoding: this.keyEncoding,
valueEncoding: this.valueEncoding
})

this.stream = this._differ(this.current, this.previous, this.range)

try {
for await (const data of this.stream) { // eslint-disable-line
this.currentMapped = this.map(this.current)
this.previousMapped = this.map(this.previous)
this.emit('update')
return { done: false, value: [this.currentMapped, this.previousMapped] }
}
} finally {
this.stream = null
}
}
} finally {
release()
}
}

async return () {
await this.close()
return { done: true }
}

async _close () {
const top = this.bee._watchers.pop()
if (top !== this) {
top.index = this.index
this.bee._watchers[top.index] = top
}

if (this.stream && !this.stream.destroying) {
this.stream.destroy()
}

this._onappend() // Continue execution being closed

await this._closeCurrent().catch(safetyCatch)
await this._closePrevious().catch(safetyCatch)

const release = await this._lock()
release()
}

destroy () {
return this.close()
}

async _closeCurrent () {
if (this.currentMapped) await this.currentMapped.close()
if (this.current) await this.current.close()
this.current = this.currentMapped = null
}

async _closePrevious () {
if (this.previousMapped) await this.previousMapped.close()
if (this.previous) await this.previous.close()
this.previous = this.previousMapped = null
}
}

function autoFlowOnUpdate (name) {
if (name === 'update') this._consume()
}

function defaultWatchMap (snapshot) {
return snapshot
}

async function leafSize (node, goLeft) {
while (node.children.length) node = await node.getChildNode(goLeft ? 0 : node.children.length - 1)
return node.keys.length
Expand Down Expand Up @@ -1340,10 +1084,6 @@ function prefixEncoding (prefix, keyEncoding) {
}
}

function defaultDiffer (currentSnap, previousSnap, opts) {
return currentSnap.createDiffStream(previousSnap.version, opts)
}

function noop () {}

module.exports = Hyperbee
Loading