diff --git a/balloon/hyper2/proof.go b/balloon/hyper2/proof.go index a26330672..9463b5f44 100644 --- a/balloon/hyper2/proof.go +++ b/balloon/hyper2/proof.go @@ -20,7 +20,7 @@ import ( "bytes" "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/balloon/hyper2/pruning2" + "github.com/bbva/qed/balloon/hyper2/pruning" "github.com/bbva/qed/hashing" "github.com/bbva/qed/log" ) @@ -53,8 +53,8 @@ func (p QueryProof) Verify(key []byte, expectedRootHash hashing.Digest) (valid b } // build a stack of operations and then interpret it to recompute the root hash - ops := pruning2.PruneToVerify(key, p.Value, p.hasher.Len()-uint16(len(p.AuditPath))) - ctx := &pruning2.Context{ + ops := pruning.PruneToVerify(key, p.Value, p.hasher.Len()-uint16(len(p.AuditPath))) + ctx := &pruning.Context{ Hasher: p.hasher, AuditPath: p.AuditPath, } diff --git a/balloon/hyper2/pruning/batch.go b/balloon/hyper2/pruning/batch.go index 10f0e758c..bb07cb42a 100644 --- a/balloon/hyper2/pruning/batch.go +++ b/balloon/hyper2/pruning/batch.go @@ -1,3 +1,19 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( @@ -35,7 +51,7 @@ func (b BatchNode) String() string { } func (b BatchNode) HasLeafAt(i int8) bool { - return b.Batch[i][b.nodeSize] == byte(1) + return len(b.Batch[i]) > 0 && b.Batch[i][b.nodeSize] == byte(1) } func (b BatchNode) AddHashAt(i int8, value []byte) { diff --git a/balloon/hyper2/pruning/insert.go b/balloon/hyper2/pruning/insert.go index b4f82c269..1b0adb6c6 100644 --- a/balloon/hyper2/pruning/insert.go +++ b/balloon/hyper2/pruning/insert.go @@ -1,3 +1,19 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( @@ -43,93 +59,79 @@ func (l Leaves) Split(index []byte) (left, right Leaves) { return l[:splitIndex], l[splitIndex:] } -type TraverseBatch func(pos *navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8) (Operation, error) +type TraverseBatch func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) -func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches BatchLoader) (Operation, error) { +func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches BatchLoader) *OperationsStack { var traverse, traverseThroughCache, traverseAfterCache TraverseBatch - traverse = func(pos *navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8) (Operation, error) { - - var err error + traverse = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { if batch == nil { - batch, err = batches.Load(pos) - if err != nil { - return nil, err - } + batch = batches.Load(pos) } - if pos.Height > cacheHeightLimit { - return traverseThroughCache(pos, leaves, batch, iBatch) + traverseThroughCache(pos, leaves, batch, iBatch, ops) + } else { + traverseAfterCache(pos, leaves, batch, iBatch, ops) } - return traverseAfterCache(pos, leaves, batch, iBatch) - } - traverseThroughCache = func(pos *navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8) (Operation, error) { + traverseThroughCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { if len(leaves) == 0 { // discarded branch if batch.HasElementAt(iBatch) { - return NewUseProvidedOp(pos, batch, iBatch), nil + ops.Push(getProvidedHash(pos, iBatch, batch)) + } else { + ops.Push(getDefaultHash(pos)) } - return NewGetDefaultOp(pos), nil + return } // at the end of a batch tree if iBatch > 0 && pos.Height%4 == 0 { - op, err := traverse(pos, leaves, nil, 0) - if err != nil { - return nil, err - } - return NewPutBatchOp(op, batch), nil + traverse(pos, leaves, nil, 0, ops) + ops.Push(updateBatchNode(pos, iBatch, batch)) + return } // on an internal node of the subtree - // we found a node in our path - if batch.HasElementAt(iBatch) { - // we found a shortcut leaf in our path - if batch.HasLeafAt(iBatch) { - // push down leaf - key, value := batch.GetLeafKVAt(iBatch) - leaves = leaves.InsertSorted(Leaf{key, value}) - batch.ResetElementAt(iBatch) - batch.ResetElementAt(2*iBatch + 1) - batch.ResetElementAt(2*iBatch + 2) - return traverse(pos, leaves, batch, iBatch) - } + // we found a node in our path and it is a shortcut leaf + if batch.HasLeafAt(iBatch) { + // push down leaf + key, value := batch.GetLeafKVAt(iBatch) + leaves = leaves.InsertSorted(Leaf{key, value}) + batch.ResetElementAt(iBatch) + batch.ResetElementAt(2*iBatch + 1) + batch.ResetElementAt(2*iBatch + 2) + traverseThroughCache(pos, leaves, batch, iBatch, ops) + return } // on an internal node with more than one leaf rightPos := pos.Right() - leftPos := pos.Left() leftLeaves, rightLeaves := leaves.Split(rightPos.Index) - left, err := traverse(&leftPos, leftLeaves, batch, 2*iBatch+1) - if err != nil { - return nil, err - } - right, err := traverse(&rightPos, rightLeaves, batch, 2*iBatch+2) - if err != nil { - return nil, err - } + traverseThroughCache(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops) + traverseThroughCache(rightPos, rightLeaves, batch, 2*iBatch+2, ops) - op := NewInnerHashOp(pos, batch, iBatch, left, right) + ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch)) if iBatch == 0 { // it's the root of the batch tree - return NewPutBatchOp(op, batch), nil + ops.Push(putInCache(pos, batch)) } - return op, nil } - traverseAfterCache = func(pos *navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8) (Operation, error) { + traverseAfterCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { if len(leaves) == 0 { // discarded branch if batch.HasElementAt(iBatch) { - return NewUseProvidedOp(pos, batch, iBatch), nil + ops.Push(getProvidedHash(pos, iBatch, batch)) + } else { + ops.Push(getDefaultHash(pos)) } - return NewGetDefaultOp(pos), nil + return } // at the end of the main tree @@ -140,39 +142,38 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches } // create or update the leaf with a new shortcut newBatch := NewEmptyBatchNode(len(pos.Index)) - shortcut := NewMutateBatchOp( - NewShortcutLeafOp(pos, newBatch, 0, leaves[0].Index, leaves[0].Value), - newBatch, + ops.PushAll( + leafHash(pos, leaves[0].Value), + updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value), + mutateBatch(pos, newBatch), + updateBatchNode(pos, iBatch, batch), ) - return NewLeafOp(pos, batch, iBatch, shortcut), nil + return } // at the end of a subtree if iBatch > 0 && pos.Height%4 == 0 { if len(leaves) > 1 { // with more than one leaf to insert -> it's impossible to be a shortcut leaf - op, err := traverse(pos, leaves, nil, 0) - if err != nil { - return nil, err - } - return NewLeafOp(pos, batch, iBatch, op), nil + traverse(pos, leaves, nil, 0, ops) + ops.Push(updateBatchNode(pos, iBatch, batch)) + return } - // with only one leaf to insert -> add a new shortcut leaf or continue traversing + // with only one leaf to insert -> continue traversing if batch.HasElementAt(iBatch) { - // continue traversing - op, err := traverse(pos, leaves, nil, 0) - if err != nil { - return nil, err - } - return NewLeafOp(pos, batch, iBatch, op), nil + traverse(pos, leaves, nil, 0, ops) + ops.Push(updateBatchNode(pos, iBatch, batch)) + return } // nil value (no previous node stored) so create a new shortcut batch newBatch := NewEmptyBatchNode(len(pos.Index)) - shortcut := NewMutateBatchOp( - NewShortcutLeafOp(pos, newBatch, 0, leaves[0].Index, leaves[0].Value), - newBatch, + ops.PushAll( + leafHash(pos, leaves[0].Value), + updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value), + mutateBatch(pos, newBatch), + updateBatchNode(pos, iBatch, batch), ) - return NewLeafOp(pos, batch, iBatch, shortcut), nil + return } // on an internal node with only one leaf to insert @@ -180,52 +181,46 @@ func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches if len(leaves) == 1 { // we found a nil in our path -> create a shortcut leaf if !batch.HasElementAt(iBatch) { - shortcut := NewShortcutLeafOp(pos, batch, iBatch, leaves[0].Index, leaves[0].Value) - if pos.Height%4 == 0 { // at the root or at leaf of the subtree - return NewMutateBatchOp(shortcut, NewEmptyBatchNode(len(pos.Index))), nil + ops.PushAll( + leafHash(pos, leaves[0].Value), + updateBatchShortcut(pos, iBatch, batch, leaves[0].Index, leaves[0].Value), + ) + if pos.Height%4 == 0 { // at the root or at a leaf of the subtree (not necessary to check iBatch) + ops.Push(mutateBatch(pos, batch)) } - return shortcut, nil + return } - // we found a node in our path - if batch.HasElementAt(iBatch) { - // we found a shortcut leaf in our path - if batch.HasLeafAt(iBatch) { - // push down leaf - key, value := batch.GetLeafKVAt(iBatch) - leaves = leaves.InsertSorted(Leaf{key, value}) - batch.ResetElementAt(iBatch) - batch.ResetElementAt(2*iBatch + 1) - batch.ResetElementAt(2*iBatch + 2) - return traverse(pos, leaves, batch, iBatch) - } + // we found a node in our path and itis a shortcut leaf + if batch.HasLeafAt(iBatch) { + // push down leaf + key, value := batch.GetLeafKVAt(iBatch) + leaves = leaves.InsertSorted(Leaf{key, value}) + batch.ResetElementAt(iBatch) + batch.ResetElementAt(2*iBatch + 1) + batch.ResetElementAt(2*iBatch + 2) + traverseAfterCache(pos, leaves, batch, iBatch, ops) + return } } // on an internal node with more than one leaf rightPos := pos.Right() - leftPos := pos.Left() leftLeaves, rightLeaves := leaves.Split(rightPos.Index) - left, err := traverse(&leftPos, leftLeaves, batch, 2*iBatch+1) - if err != nil { - return nil, err - } - right, err := traverse(&rightPos, rightLeaves, batch, 2*iBatch+2) - if err != nil { - return nil, err - } + traverseAfterCache(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops) + traverseAfterCache(rightPos, rightLeaves, batch, 2*iBatch+2, ops) - op := NewInnerHashOp(pos, batch, iBatch, left, right) + ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch)) if iBatch == 0 { // at root node -> mutate batch - return NewMutateBatchOp(op, batch), nil + ops.Push(mutateBatch(pos, batch)) } - return op, nil } + ops := NewOperationsStack() leaves := make(Leaves, 0) leaves = leaves.InsertSorted(Leaf{index, value}) - root := navigation.NewRootPosition(uint16(len(index) * 8)) - return traverse(&root, leaves, nil, 0) + traverse(navigation.NewRootPosition(uint16(len(index)*8)), leaves, nil, 0, ops) + return ops } diff --git a/balloon/hyper2/pruning/insert_test.go b/balloon/hyper2/pruning/insert_test.go index b993a0184..e2be56a4f 100644 --- a/balloon/hyper2/pruning/insert_test.go +++ b/balloon/hyper2/pruning/insert_test.go @@ -1,8 +1,28 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( "testing" + "github.com/bbva/qed/balloon/cache" + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/hashing" + "github.com/bbva/qed/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,7 +33,7 @@ func TestPruneToInsert(t *testing.T) { index, value []byte cachedBatches map[string][]byte storedBatches map[string][]byte - expectedOp Operation + expectedOps []op }{ { // insert index = 0 on empty tree @@ -21,122 +41,271 @@ func TestPruneToInsert(t *testing.T) { value: []byte{0}, cachedBatches: map[string][]byte{}, storedBatches: map[string][]byte{}, - expectedOp: putBatch( - inner(pos(0, 8), 0, []byte{0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 7), 1, []byte{0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 6), 3, []byte{0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 5), 7, []byte{0x00, 0x00, 0x00, 0x00}, - leaf(pos(0, 4), 15, []byte{0x00, 0x00, 0x00, 0x00}, - mutate( - shortcut(pos(0, 4), 0, []byte{0x00, 0x00, 0x00, 0x00}, - []byte{0}, []byte{0}, - ), - []byte{0x00, 0x00, 0x00, 0x00}, - ), - ), - getDefault(pos(16, 4)), - ), - getDefault(pos(32, 5)), - ), - getDefault(pos(64, 6)), - ), - getDefault(pos(128, 7)), - ), - []byte{0x00, 0x00, 0x00, 0x00}, - ), + expectedOps: []op{ + {PutInCacheCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(0, 8)}, + {InnerHashCode, pos(0, 8)}, + {GetDefaultHashCode, pos(128, 7)}, + {UpdateBatchNodeCode, pos(0, 7)}, + {InnerHashCode, pos(0, 7)}, + {GetDefaultHashCode, pos(64, 6)}, + {UpdateBatchNodeCode, pos(0, 6)}, + {InnerHashCode, pos(0, 6)}, + {GetDefaultHashCode, pos(32, 5)}, + {UpdateBatchNodeCode, pos(0, 5)}, + {InnerHashCode, pos(0, 5)}, + {GetDefaultHashCode, pos(16, 4)}, + {UpdateBatchNodeCode, pos(0, 4)}, + {MutateBatchCode, pos(0, 4)}, + {UpdateBatchShortcutCode, pos(0, 4)}, + {LeafHashCode, pos(0, 4)}, + }, }, { // update index = 0 on tree with only one leaf index: []byte{0}, value: []byte{0}, cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, }, storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - }, - expectedOp: putBatch( - inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - mutate( - shortcut(pos(0, 4), 0, []byte{0x00, 0x00, 0x00, 0x00}, - []byte{0}, []byte{0}, - ), - []byte{0x00, 0x00, 0x00, 0x00}, // <-- the batch has been reset - ), - ), - getDefault(pos(16, 4)), - ), - getDefault(pos(32, 5)), - ), - getDefault(pos(64, 6)), - ), - getDefault(pos(128, 7)), - ), - []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - ), + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {PutInCacheCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(0, 8)}, + {InnerHashCode, pos(0, 8)}, + {GetDefaultHashCode, pos(128, 7)}, + {UpdateBatchNodeCode, pos(0, 7)}, + {InnerHashCode, pos(0, 7)}, + {GetDefaultHashCode, pos(64, 6)}, + {UpdateBatchNodeCode, pos(0, 6)}, + {InnerHashCode, pos(0, 6)}, + {GetDefaultHashCode, pos(32, 5)}, + {UpdateBatchNodeCode, pos(0, 5)}, + {InnerHashCode, pos(0, 5)}, + {GetDefaultHashCode, pos(16, 4)}, + {UpdateBatchNodeCode, pos(0, 4)}, + {MutateBatchCode, pos(0, 4)}, + {UpdateBatchShortcutCode, pos(0, 4)}, + {LeafHashCode, pos(0, 4)}, + }, }, { - // insert key = 1 on tree with 1 leaf (index: 0, value: 0) + // insert index=1 on tree with 1 leaf (index: 0, value: 0) // it should push down the previous leaf to the last level index: []byte{1}, value: []byte{1}, cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, }, storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - }, - expectedOp: putBatch( - inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - mutate( - inner(pos(0, 4), 0, []byte{0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 3), 1, []byte{0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 2), 3, []byte{0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 1), 7, []byte{0x00, 0x00, 0x00, 0x00}, - leaf(pos(0, 0), 15, []byte{0x00, 0x00, 0x00, 0x00}, - mutate( - shortcut(pos(0, 0), 0, []byte{0x00, 0x00, 0x00, 0x00}, - []byte{0}, []byte{0}, - ), - []byte{0x00, 0x00, 0x00, 0x00}, // new batch - ), - ), - leaf(pos(1, 0), 16, []byte{0x00, 0x00, 0x00, 0x00}, - mutate( - shortcut(pos(1, 0), 0, []byte{0x00, 0x00, 0x00, 0x00}, - []byte{1}, []byte{1}, - ), - []byte{0x00, 0x00, 0x00, 0x00}, // new batch - ), - ), - ), - getDefault(pos(2, 1)), - ), - getDefault(pos(4, 2)), - ), - getDefault(pos(8, 3)), - ), - []byte{0x00, 0x00, 0x00, 0x00}, // reset previous shortcut - ), - ), - getDefault(pos(16, 4)), - ), - getDefault(pos(32, 5)), - ), - getDefault(pos(64, 6)), - ), - getDefault(pos(128, 7)), - ), - []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // does not change - ), + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {PutInCacheCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(0, 8)}, + {InnerHashCode, pos(0, 8)}, + {GetDefaultHashCode, pos(128, 7)}, + {UpdateBatchNodeCode, pos(0, 7)}, + {InnerHashCode, pos(0, 7)}, + {GetDefaultHashCode, pos(64, 6)}, + {UpdateBatchNodeCode, pos(0, 6)}, + {InnerHashCode, pos(0, 6)}, + {GetDefaultHashCode, pos(32, 5)}, + {UpdateBatchNodeCode, pos(0, 5)}, + {InnerHashCode, pos(0, 5)}, + {GetDefaultHashCode, pos(16, 4)}, + {UpdateBatchNodeCode, pos(0, 4)}, + {MutateBatchCode, pos(0, 4)}, // reset previous shortcut + {UpdateBatchNodeCode, pos(0, 4)}, + {InnerHashCode, pos(0, 4)}, + {GetDefaultHashCode, pos(8, 3)}, + {UpdateBatchNodeCode, pos(0, 3)}, + {InnerHashCode, pos(0, 3)}, + {GetDefaultHashCode, pos(4, 2)}, + {UpdateBatchNodeCode, pos(0, 2)}, + {InnerHashCode, pos(0, 2)}, + {GetDefaultHashCode, pos(2, 1)}, + {UpdateBatchNodeCode, pos(0, 1)}, + {InnerHashCode, pos(0, 1)}, + {UpdateBatchNodeCode, pos(1, 0)}, + {MutateBatchCode, pos(1, 0)}, // new batch + {UpdateBatchShortcutCode, pos(1, 0)}, + {LeafHashCode, pos(1, 0)}, + {UpdateBatchNodeCode, pos(0, 0)}, + {MutateBatchCode, pos(0, 0)}, // new batch + {UpdateBatchShortcutCode, pos(0, 0)}, + {LeafHashCode, pos(0, 0)}, + }, + }, + { + // insert index=8 on tree with 1 leaf (index: 0, value: 0) + // it should push down the previous leaf to the next subtree + index: []byte{8}, + value: []byte{8}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {PutInCacheCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(0, 8)}, + {InnerHashCode, pos(0, 8)}, + {GetDefaultHashCode, pos(128, 7)}, + {UpdateBatchNodeCode, pos(0, 7)}, + {InnerHashCode, pos(0, 7)}, + {GetDefaultHashCode, pos(64, 6)}, + {UpdateBatchNodeCode, pos(0, 6)}, + {InnerHashCode, pos(0, 6)}, + {GetDefaultHashCode, pos(32, 5)}, + {UpdateBatchNodeCode, pos(0, 5)}, + {InnerHashCode, pos(0, 5)}, + {GetDefaultHashCode, pos(16, 4)}, + {UpdateBatchNodeCode, pos(0, 4)}, + {MutateBatchCode, pos(0, 4)}, // reset previous shortcut + {UpdateBatchNodeCode, pos(0, 4)}, + {InnerHashCode, pos(0, 4)}, + {UpdateBatchShortcutCode, pos(8, 3)}, + {LeafHashCode, pos(8, 3)}, + {UpdateBatchShortcutCode, pos(0, 3)}, + {LeafHashCode, pos(0, 3)}, + }, + }, + { + + // insert index=12 on tree with 2 leaves ([index:0, value:0], [index:8, value:8]) + // it should push down the leaf with index=8 to the next level + index: []byte{12}, + value: []byte{12}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x08, 0x00, // iBatch 0 -> hash=0x08 + 0x08, 0x00, // iBatch 1 -> hash=0x08 + 0x08, 0x00, // iBatch 3 -> hash=0x08 + 0x08, 0x00, // iBatch 7 -> hash=0x08 + 0x08, 0x00, // iBatch 15 -> hash=0x08 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xfe, 0x00, 0x00, 0x00, // bitmap: 11111110 00000000 00000000 00000000 + 0x08, 0x00, // iBatch 0 -> hash=0x08 + 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) + 0x08, 0x01, // iBatch 2 -> hash=0x08 (shortcut index=8) + 0x00, 0x02, // iBatch 3 -> key=0x00 + 0x00, 0x02, // iBatch 4 -> value=0x00 + 0x08, 0x02, // iBatch 5 -> key=0x08 + 0x08, 0x02, // iBatch 6 -> value=0x08 + }, + }, + expectedOps: []op{ + {PutInCacheCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(0, 8)}, + {InnerHashCode, pos(0, 8)}, + {GetDefaultHashCode, pos(128, 7)}, + {UpdateBatchNodeCode, pos(0, 7)}, + {InnerHashCode, pos(0, 7)}, + {GetDefaultHashCode, pos(64, 6)}, + {UpdateBatchNodeCode, pos(0, 6)}, + {InnerHashCode, pos(0, 6)}, + {GetDefaultHashCode, pos(32, 5)}, + {UpdateBatchNodeCode, pos(0, 5)}, + {InnerHashCode, pos(0, 5)}, + {GetDefaultHashCode, pos(16, 4)}, + {UpdateBatchNodeCode, pos(0, 4)}, + {MutateBatchCode, pos(0, 4)}, + {UpdateBatchNodeCode, pos(0, 4)}, + {InnerHashCode, pos(0, 4)}, + {UpdateBatchNodeCode, pos(8, 3)}, + {InnerHashCode, pos(8, 3)}, + {UpdateBatchShortcutCode, pos(12, 2)}, + {LeafHashCode, pos(12, 2)}, + {UpdateBatchShortcutCode, pos(8, 2)}, + {LeafHashCode, pos(8, 2)}, + {GetProvidedHashCode, pos(0, 3)}, + }, + }, + { + // insert index=128 on tree with one leaf ([index:0, value:0] + index: []byte{128}, + value: []byte{128}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {PutInCacheCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(0, 8)}, + {InnerHashCode, pos(0, 8)}, + {UpdateBatchNodeCode, pos(128, 7)}, + {InnerHashCode, pos(128, 7)}, + {GetDefaultHashCode, pos(192, 6)}, + {UpdateBatchNodeCode, pos(128, 6)}, + {InnerHashCode, pos(128, 6)}, + {GetDefaultHashCode, pos(160, 5)}, + {UpdateBatchNodeCode, pos(128, 5)}, + {InnerHashCode, pos(128, 5)}, + {GetDefaultHashCode, pos(144, 4)}, + {UpdateBatchNodeCode, pos(128, 4)}, + {MutateBatchCode, pos(128, 4)}, + {UpdateBatchShortcutCode, pos(128, 4)}, + {LeafHashCode, pos(128, 4)}, + {GetProvidedHashCode, pos(0, 7)}, + }, }, } @@ -145,181 +314,373 @@ func TestPruneToInsert(t *testing.T) { for i, c := range testCases { loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - prunedOp, err := PruneToInsert(c.index, c.value, cacheHeightLimit, loader) - require.NoError(t, err) - assert.Equalf(t, c.expectedOp, prunedOp, "The pruned operation should match for test case %d", i) + prunedOps := PruneToInsert(c.index, c.value, cacheHeightLimit, loader).List() + require.Truef(t, len(c.expectedOps) == len(prunedOps), "The size of the pruned ops should match the expected for test case %d", i) + for j := 0; j < len(prunedOps); j++ { + assert.Equalf(t, c.expectedOps[j].Code, prunedOps[j].Code, "The pruned operation's code should match for test case %d", i) + assert.Equalf(t, c.expectedOps[j].Pos, prunedOps[j].Pos, "The pruned operation's position should match for test case %d", i) + } } -} -// // insert Key = 1 after 0 +} -// PutBatchOp( -// InnerHashOp(pos(0, 8), 0, -// InnerHashOp(pos(0, 7), 1, -// InnerHashOp(pos(0, 6), 3 -// InnerHashOp(pos(0, 5), 7, -// LeafOp(pos(0, 4), 15, -// MutateBatchOp( -// InnerHashOp(pos(0, 4), 0), -// InnerHashOp(pos(0, 3), 1, -// InnerHashOp(pos(0, 2), 3, -// InnerHashOp(pos(0, 1), 7, -// LeafOp(pos(0, 0), 15, -// MutateBatchOp( -// ShortcutLeafOp(pos(0, 4), 0, Key[0], Value), -// batch, -// ), -// ), -// LeafOp(pos(1, 0), 15, -// MutateBatchOp( -// ShortcutLeafOp(pos(0, 4), 0, Key[1], Value), -// batch, -// ), -// ), -// ), -// GetDefaultOp(pos(2, 1)), -// ), -// GetDefaultOp(pos(4, 2)), -// ), -// GetDefaultOp(pos(8, 3)), -// ), -// batch, -// ), -// ), -// GetDefaultOp(pos(16, 4)), -// ), -// GetDefaultOp(pos(32, 5)), -// ), -// GetDefaultOp(pos(64, 6)), -// ), -// GetDefaultOp(pos(128, 7)), -// ), -// batch, -// ) +func TestInsertInterpretation(t *testing.T) { -// // insert Key = 0 after nil + testCases := []struct { + index, value []byte + cachedBatches map[string][]byte + storedBatches map[string][]byte + expectedMutations []*storage.Mutation + expectedElements []*cachedElement + }{ + { + // insert index = 0 on empty tree + index: []byte{0}, + value: []byte{0}, + cachedBatches: map[string][]byte{}, + storedBatches: map[string][]byte{}, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 4).Bytes(), + Value: []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + }, + expectedElements: []*cachedElement{ + { + Pos: pos(0, 8), + Value: []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + }, + }, + { + // update index = 0 on tree with only one leaf + index: []byte{0}, + value: []byte{0}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 4).Bytes(), + Value: []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + }, + expectedElements: []*cachedElement{ + { + Pos: pos(0, 8), + Value: []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + }, + }, + { + // insert index=1 on tree with 1 leaf (index: 0, value: 0) + // it should push down the previous leaf to the last level + index: []byte{1}, + value: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(1, 0).Bytes(), + Value: []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x01, 0x01, // iBatch 0 -> hash=0x01 (shortcut index=0) + 0x01, 0x02, // iBatch 1 -> key=0x01 + 0x01, 0x02, // iBatch 2 -> value=0x01 + }, + }, + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 0).Bytes(), + Value: []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 4).Bytes(), + Value: []byte{ + 0xd1, 0x01, 0x80, 0x00, // bitmap: 11010001 00000001 10000000 00000000 + 0x01, 0x00, // iBatch 0 -> hash=0x01 + 0x01, 0x00, // iBatch 1 -> hash=0x01 + 0x01, 0x00, // iBatch 3 -> hash=0x01 + 0x01, 0x00, // iBatch 7 -> hash=0x01 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + 0x01, 0x00, // iBatch 16 -> hash=0x01 + }, + }, + }, + expectedElements: []*cachedElement{ + { + Pos: pos(0, 8), + Value: []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x01, 0x00, // iBatch 0 -> hash=0x01 + 0x01, 0x00, // iBatch 1 -> hash=0x01 + 0x01, 0x00, // iBatch 3 -> hash=0x01 + 0x01, 0x00, // iBatch 7 -> hash=0x01 + 0x01, 0x00, // iBatch 15 -> hash=0x01 + }, + }, + }, + }, + { + // insert index=8 on tree with 1 leaf (index: 0, value: 0) + // it should push down the previous leaf to the next subtree + index: []byte{8}, + value: []byte{8}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 4).Bytes(), + Value: []byte{ + 0xfe, 0x00, 0x00, 0x00, // bitmap: 11111110 00000000 00000000 00000000 + 0x08, 0x00, // iBatch 0 -> hash=0x08 + 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) + 0x08, 0x01, // iBatch 2 -> hash=0x08 (shortcut index=8) + 0x00, 0x02, // iBatch 3 -> key=0x00 + 0x00, 0x02, // iBatch 4 -> value=0x00 + 0x08, 0x02, // iBatch 5 -> key=0x08 + 0x08, 0x02, // iBatch 6 -> value=0x08 + }, + }, + }, + expectedElements: []*cachedElement{ + { + Pos: pos(0, 8), + Value: []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x08, 0x00, // iBatch 0 -> hash=0x08 + 0x08, 0x00, // iBatch 1 -> hash=0x08 + 0x08, 0x00, // iBatch 3 -> hash=0x08 + 0x08, 0x00, // iBatch 7 -> hash=0x08 + 0x08, 0x00, // iBatch 15 -> hash=0x08 + }, + }, + }, + }, + { + // insert index=12 on tree with 2 leaves ([index:0, value:0], [index:8, value:8]) + // it should push down the leaf with index=8 to the next level + index: []byte{12}, + value: []byte{12}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x08, 0x00, // iBatch 0 -> hash=0x08 + 0x08, 0x00, // iBatch 1 -> hash=0x08 + 0x08, 0x00, // iBatch 3 -> hash=0x08 + 0x08, 0x00, // iBatch 7 -> hash=0x08 + 0x08, 0x00, // iBatch 15 -> hash=0x08 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xfe, 0x00, 0x00, 0x00, // bitmap: 11111110 00000000 00000000 00000000 + 0x08, 0x00, // iBatch 0 -> hash=0x08 + 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) + 0x08, 0x01, // iBatch 2 -> hash=0x08 (shortcut index=8) + 0x00, 0x02, // iBatch 3 -> key=0x00 + 0x00, 0x02, // iBatch 4 -> value=0x00 + 0x08, 0x02, // iBatch 5 -> key=0x08 + 0x08, 0x02, // iBatch 6 -> value=0x08 + }, + }, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(0, 4).Bytes(), + Value: []byte{ + 0xfe, 0x1e, 0x00, 0x00, // bitmap: 11111110 00011110 00000000 00000000 + 0x04, 0x00, // iBatch 0 -> hash=0x08 + 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) + 0x04, 0x00, // iBatch 2 -> hash=0x04 + 0x00, 0x02, // iBatch 3 -> key=0x00 + 0x00, 0x02, // iBatch 4 -> value=0x00 + 0x08, 0x01, // iBatch 5 -> hash=0x08 (shortcut index=8) + 0x0c, 0x01, // iBatch 6 -> hash=0x0c (shortcut index=12) + 0x08, 0x02, // iBatch 11 -> key=0x08 + 0x08, 0x02, // iBatch 12 -> value=0x08 + 0x0c, 0x02, // iBatch 13 -> key=0x0c + 0x0c, 0x02, // iBatch 14 -> value=0x0c + }, + }, + }, + expectedElements: []*cachedElement{ + { + Pos: pos(0, 8), + Value: []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x04, 0x00, // iBatch 0 -> hash=0x04 + 0x04, 0x00, // iBatch 1 -> hash=0x04 + 0x04, 0x00, // iBatch 3 -> hash=0x04 + 0x04, 0x00, // iBatch 7 -> hash=0x04 + 0x04, 0x00, // iBatch 15 -> hash=0x04 + }, + }, + }, + }, + { + // insert index=128 on tree with one leaf ([index:0, value:0] + index: []byte{128}, + value: []byte{128}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedMutations: []*storage.Mutation{ + { + Prefix: storage.HyperCachePrefix, + Key: pos(128, 4).Bytes(), + Value: []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x80, 0x01, // iBatch 0 -> hash=0x80 (shortcut index=128) + 0x80, 0x02, // iBatch 1 -> key=0x80 + 0x80, 0x02, // iBatch 2 -> value=0x80 + }, + }, + }, + expectedElements: []*cachedElement{ + { + Pos: pos(0, 8), + Value: []byte{ + 0xf5, 0x11, 0x01, 0x00, // bitmap: 11110101 00010001 00000001 00000000 + 0x80, 0x00, // iBatch 0 -> hash=0x80 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x80, 0x00, // iBatch 2 -> hash=0x80 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x80, 0x00, // iBatch 5 -> hash=0x80 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x80, 0x00, // iBatch 11 -> hash=0x80 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + 0x80, 0x00, // iBatch 23 -> hash=0x80 + }, + }, + }, + }, + } -// PutBatchOp( -// InnerHashOp(pos(0, 8), 0, -// InnerHashOp(pos(0, 7), 1, -// InnerHashOp(pos(0, 6), 3 -// InnerHashOp(pos(0, 5), 7, -// LeafOp(pos(0, 4), 15, -// MutateBatchOp( -// ShortcutLeafOp(pos(0, 4), 0, Key, Value), -// batch, -// ), -// ), -// GetDefaultOp(pos(16, 4)), -// ), -// GetDefaultOp(pos(32, 5)), -// ), -// GetDefaultOp(pos(64, 6)), -// ), -// GetDefaultOp(pos(128, 7)), -// ), -// batch, -// ) + batchLevels := uint16(1) + cacheHeightLimit := batchLevels * 4 + defaultHashes := []hashing.Digest{{0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}} -// // insert Key = 8 after 0 + for i, c := range testCases { + cache := cache.NewFakeCache([]byte{0x0}) + batches := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) -// PutBatchOp( -// InnerHashOp(pos(0, 8), 0, -// InnerHashOp(pos(0, 7), 1, -// InnerHashOp(pos(0, 6), 3 -// InnerHashOp(pos(0, 5), 7, -// LeafOp(pos(0, 4), 15, -// MutateBatchOp( -// InnerHashOp(pos(0, 4), 0), -// ShortcutLeafOp(pos(0, 3), 1, Key[0], Value), // pushed down -// ShortcutLeafOp(pos(8, 3), 2, Key[8], Value), // new -// ), -// batch, -// ), -// ), -// GetDefaultOp(pos(16, 4)), -// ), -// GetDefaultOp(pos(32, 5)), -// ), -// GetDefaultOp(pos(64, 6)), -// ), -// GetDefaultOp(pos(128, 7)), -// ), -// batch, -// ) + ops := PruneToInsert(c.index, c.value, cacheHeightLimit, batches) + ctx := &Context{ + Hasher: hashing.NewFakeXorHasher(), + Cache: cache, + DefaultHashes: defaultHashes, + Mutations: make([]*storage.Mutation, 0), + } -// // insert Key = 12 after 0 and 8 + ops.Pop().Interpret(ops, ctx) -// PutBatchOp( -// InnerHashOp(pos(0, 8), 0, -// InnerHashOp(pos(0, 7), 1, -// InnerHashOp(pos(0, 6), 3 -// InnerHashOp(pos(0, 5), 7, -// LeafOp(pos(0, 4), 15, -// MutateBatchOp( -// InnerHashOp(pos(0, 4), 0), -// UseProvidedOp(pos(0, 3), 1), // provided -// InnerHashOp(pos(8, 3), 2, -// ShortcutLeafOp(pos(8, 2), 5, Key[8], Value), // pushed down -// ShortcutLeafOp(pos(12, 2), 6, Key[12], Value), -// ), -// ), -// batch, -// ), -// ), -// GetDefaultOp(pos(16, 4)), -// ), -// GetDefaultOp(pos(32, 5)), -// ), -// GetDefaultOp(pos(64, 6)), -// ), -// GetDefaultOp(pos(128, 7)), -// ), -// batch, -// ) + assert.ElementsMatchf(t, c.expectedMutations, ctx.Mutations, "Mutation error in test case %d", i) + for _, e := range c.expectedElements { + v, _ := cache.Get(e.Pos.Bytes()) + assert.Equalf(t, e.Value, v, "The cached element %v should be cached in test case %d", e, i) + } + } -// // insert Key = 1 after 0 +} -// PutBatchOp( -// InnerHashOp(pos(0, 8), 0, -// InnerHashOp(pos(0, 7), 1, -// InnerHashOp(pos(0, 6), 3 -// InnerHashOp(pos(0, 5), 7, -// LeafOp(pos(0, 4), 15, -// MutateBatchOp( -// InnerHashOp(pos(0, 4), 0), -// InnerHashOp(pos(0, 3), 1, -// InnerHashOp(pos(0, 2), 3, -// InnerHashOp(pos(0, 1), 7, -// LeafOp(pos(0, 0), 15, -// MutateBatchOp( -// ShortcutLeafOp(pos(0, 4), 0, Key[0], Value), -// batch, -// ), -// ), -// LeafOp(pos(1, 0), 15, -// MutateBatchOp( -// ShortcutLeafOp(pos(0, 4), 0, Key[1], Value), -// batch, -// ), -// ), -// ), -// GetDefaultOp(pos(2, 1)), -// ), -// GetDefaultOp(pos(4, 2)), -// ), -// GetDefaultOp(pos(8, 3)), -// ), -// batch, -// ), -// ), -// GetDefaultOp(pos(16, 4)), -// ), -// GetDefaultOp(pos(32, 5)), -// ), -// GetDefaultOp(pos(64, 6)), -// ), -// GetDefaultOp(pos(128, 7)), -// ), -// batch, -// ) +type cachedElement struct { + Pos navigation.Position + Value []byte +} diff --git a/balloon/hyper2/pruning/loader.go b/balloon/hyper2/pruning/loader.go index 7153c8f4f..51810aa62 100644 --- a/balloon/hyper2/pruning/loader.go +++ b/balloon/hyper2/pruning/loader.go @@ -1,13 +1,31 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( + "github.com/bbva/qed/log" + "github.com/bbva/qed/balloon/cache" "github.com/bbva/qed/balloon/hyper2/navigation" "github.com/bbva/qed/storage" ) type BatchLoader interface { - Load(pos *navigation.Position) (*BatchNode, error) + Load(pos navigation.Position) *BatchNode } // TODO maybe use a function @@ -25,31 +43,30 @@ func NewDefaultBatchLoader(store storage.Store, cache cache.Cache, cacheHeightLi } } -// TODO remove errors and panic -func (l DefaultBatchLoader) Load(pos *navigation.Position) (*BatchNode, error) { +func (l DefaultBatchLoader) Load(pos navigation.Position) *BatchNode { if pos.Height > l.cacheHeightLimit { return l.loadBatchFromCache(pos) } return l.loadBatchFromStore(pos) } -func (l DefaultBatchLoader) loadBatchFromCache(pos *navigation.Position) (*BatchNode, error) { +func (l DefaultBatchLoader) loadBatchFromCache(pos navigation.Position) *BatchNode { value, ok := l.cache.Get(pos.Bytes()) if !ok { - return NewEmptyBatchNode(len(pos.Index)), nil + return NewEmptyBatchNode(len(pos.Index)) } batch := ParseBatchNode(len(pos.Index), value) - return batch, nil + return batch } -func (l DefaultBatchLoader) loadBatchFromStore(pos *navigation.Position) (*BatchNode, error) { +func (l DefaultBatchLoader) loadBatchFromStore(pos navigation.Position) *BatchNode { kv, err := l.store.Get(storage.HyperCachePrefix, pos.Bytes()) if err != nil { if err == storage.ErrKeyNotFound { - return NewEmptyBatchNode(len(pos.Index)), nil + return NewEmptyBatchNode(len(pos.Index)) } - return nil, err + log.Fatalf("Oops, something went wrong. Unable to load batch: %v", err) } batch := ParseBatchNode(len(pos.Index), kv.Value) - return batch, nil + return batch } diff --git a/balloon/hyper2/pruning/operation.go b/balloon/hyper2/pruning/operation.go index 350dc76d9..e0ba077a5 100644 --- a/balloon/hyper2/pruning/operation.go +++ b/balloon/hyper2/pruning/operation.go @@ -1,229 +1,183 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( - "fmt" - + "github.com/bbva/qed/balloon/cache" "github.com/bbva/qed/balloon/hyper2/navigation" "github.com/bbva/qed/hashing" + "github.com/bbva/qed/log" + "github.com/bbva/qed/storage" ) -type Operation interface { - Accept(visitor OpVisitor) hashing.Digest - String() string - Position() *navigation.Position -} - -type OpVisitor interface { // THROW ERRORS? - VisitShortcutLeafOp(op ShortcutLeafOp) hashing.Digest - VisitLeafOp(op LeafOp) hashing.Digest - VisitInnerHashOp(op InnerHashOp) hashing.Digest - VisitGetDefaultOp(op GetDefaultOp) hashing.Digest - VisitUseProvidedOp(op UseProvidedOp) hashing.Digest - VisitPutBatchOp(op PutBatchOp) hashing.Digest - VisitMutateBatchOp(op MutateBatchOp) hashing.Digest - VisitCollectOp(op CollectOp) hashing.Digest -} - -type ShortcutLeafOp struct { - pos *navigation.Position - Batch *BatchNode - Idx int8 - Key, Value []byte -} - -type LeafOp struct { - pos *navigation.Position - Batch *BatchNode - Idx int8 - Operation -} - -type InnerHashOp struct { - pos *navigation.Position - Batch *BatchNode - Idx int8 - Left, Right Operation -} - -type GetDefaultOp struct { - pos *navigation.Position -} - -type UseProvidedOp struct { - pos *navigation.Position - Batch *BatchNode - Idx int8 -} - -type PutBatchOp struct { - Operation - Batch *BatchNode -} +type Context struct { + Hasher hashing.Hasher + Cache cache.ModifiableCache + DefaultHashes []hashing.Digest + Mutations []*storage.Mutation + AuditPath navigation.AuditPath +} + +type OperationCode int + +const ( + LeafHashCode OperationCode = iota + InnerHashCode + UpdateBatchNodeCode + UpdateBatchShortcutCode + GetDefaultHashCode + GetProvidedHashCode + PutInCacheCode + MutateBatchCode + CollectHashCode + GetFromPathCode + NoOpCode +) -type MutateBatchOp struct { - Operation - Batch *BatchNode -} +type Interpreter func(ops *OperationsStack, c *Context) hashing.Digest -type CollectOp struct { - Operation +type Operation struct { + Code OperationCode + Pos navigation.Position + Interpret Interpreter } -func NewShortcutLeafOp(pos *navigation.Position, batch *BatchNode, iBatch int8, key, value []byte) *ShortcutLeafOp { - return &ShortcutLeafOp{ - pos: pos, - Batch: batch, - Idx: iBatch, - Key: key, - Value: value, +func leafHash(pos navigation.Position, value []byte) *Operation { + return &Operation{ + Code: LeafHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return c.Hasher.Salted(pos.Bytes(), value) + }, } } -func (o ShortcutLeafOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitShortcutLeafOp(o) -} - -func (o ShortcutLeafOp) Position() *navigation.Position { - return o.pos -} - -func (o ShortcutLeafOp) String() string { - return fmt.Sprintf("ShortcutLeafOp(%v)(%d)[ %x - %x ]", o.pos, o.Idx, o.Key, o.Value) -} - -func NewLeafOp(pos *navigation.Position, batch *BatchNode, iBatch int8, op Operation) *LeafOp { - return &LeafOp{ - pos: pos, - Batch: batch, - Idx: iBatch, - Operation: op, +func innerHash(pos navigation.Position) *Operation { + return &Operation{ + Code: InnerHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + leftHash := ops.Pop().Interpret(ops, c) + rightHash := ops.Pop().Interpret(ops, c) + return c.Hasher.Salted(pos.Bytes(), leftHash, rightHash) + }, } } -func (o LeafOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitLeafOp(o) -} - -func (o LeafOp) Position() *navigation.Position { - return o.pos -} - -func (o LeafOp) String() string { - return fmt.Sprintf("LeafOp(%v)(%d)[ %v ]", o.pos, o.Idx, o.Operation) -} - -func NewInnerHashOp(pos *navigation.Position, batch *BatchNode, iBatch int8, left, right Operation) *InnerHashOp { - return &InnerHashOp{ - pos: pos, - Batch: batch, - Idx: iBatch, - Left: left, - Right: right, +func updateBatchNode(pos navigation.Position, idx int8, batch *BatchNode) *Operation { + return &Operation{ + Code: UpdateBatchNodeCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + batch.AddHashAt(idx, hash) + return hash + }, } } -func (o InnerHashOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitInnerHashOp(o) -} - -func (o InnerHashOp) Position() *navigation.Position { - return o.pos -} - -func (o InnerHashOp) String() string { - return fmt.Sprintf("InnerHashOp(%v)(%d)[ %v | %v ]", o.pos, o.Idx, o.Left, o.Right) -} - -func NewGetDefaultOp(pos *navigation.Position) *GetDefaultOp { - return &GetDefaultOp{ - pos: pos, +func updateBatchShortcut(pos navigation.Position, idx int8, batch *BatchNode, key, value []byte) *Operation { + return &Operation{ + Code: UpdateBatchShortcutCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + batch.AddLeafAt(idx, hash, key, value) + return hash + }, } } -func (o GetDefaultOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitGetDefaultOp(o) -} - -func (o GetDefaultOp) Position() *navigation.Position { - return o.pos -} - -func (o GetDefaultOp) String() string { - return fmt.Sprintf("GetDefaultOp(%v)", o.pos) -} - -func NewUseProvidedOp(pos *navigation.Position, batch *BatchNode, iBatch int8) *UseProvidedOp { - return &UseProvidedOp{ - pos: pos, - Batch: batch, - Idx: iBatch, +func getDefaultHash(pos navigation.Position) *Operation { + return &Operation{ + Code: GetDefaultHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return c.DefaultHashes[pos.Height] + }, } } -func (o UseProvidedOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitUseProvidedOp(o) -} - -func (o UseProvidedOp) Position() *navigation.Position { - return o.pos -} - -func (o UseProvidedOp) String() string { - return fmt.Sprintf("UseProvidedOp(%v)(%d)", o.pos, o.Idx) -} - -func NewPutBatchOp(op Operation, batch *BatchNode) *PutBatchOp { - return &PutBatchOp{ - Operation: op, - Batch: batch, +func getProvidedHash(pos navigation.Position, idx int8, batch *BatchNode) *Operation { + return &Operation{ + Code: GetProvidedHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return batch.GetElementAt(idx) + }, } } -func (o PutBatchOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitPutBatchOp(o) -} - -func (o PutBatchOp) Position() *navigation.Position { - return o.Operation.Position() -} - -func (o PutBatchOp) String() string { - return fmt.Sprintf("PutBatchOp( %v )[ %v ]", o.Operation, o.Batch) -} - -func NewMutateBatchOp(op Operation, batch *BatchNode) *MutateBatchOp { - return &MutateBatchOp{ - Operation: op, - Batch: batch, +func putInCache(pos navigation.Position, batch *BatchNode) *Operation { + return &Operation{ + Code: PutInCacheCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + c.Cache.Put(pos.Bytes(), batch.Serialize()) + return hash + }, } } -func (o MutateBatchOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitMutateBatchOp(o) -} - -func (o MutateBatchOp) Position() *navigation.Position { - return o.Operation.Position() -} - -func (o MutateBatchOp) String() string { - return fmt.Sprintf("MutateBatchOp( %v )[ %v ]", o.Operation, o.Batch) -} - -func NewCollectOp(op Operation) *CollectOp { - return &CollectOp{ - Operation: op, +func mutateBatch(pos navigation.Position, batch *BatchNode) *Operation { + return &Operation{ + Code: MutateBatchCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + c.Mutations = append(c.Mutations, storage.NewMutation(storage.HyperCachePrefix, pos.Bytes(), batch.Serialize())) + return hash + }, } } -func (o CollectOp) Accept(visitor OpVisitor) hashing.Digest { - return visitor.VisitCollectOp(o) +func collectHash(pos navigation.Position) *Operation { + return &Operation{ + Code: CollectHashCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash := ops.Pop().Interpret(ops, c) + c.AuditPath[pos.StringId()] = hash + return hash + }, + } } -func (o CollectOp) Position() *navigation.Position { - return o.Operation.Position() +func getFromPath(pos navigation.Position) *Operation { + return &Operation{ + Code: GetFromPathCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + hash, ok := c.AuditPath.Get(pos) + if !ok { + log.Fatalf("Oops, something went wrong. Invalid position in audit path") + } + return hash + }, + } } -func (o CollectOp) String() string { - return fmt.Sprintf("CollectOp( %v )", o.Operation) +func noOp(pos navigation.Position) *Operation { + return &Operation{ + Code: NoOpCode, + Pos: pos, + Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { + return nil + }, + } } diff --git a/balloon/hyper2/pruning/search.go b/balloon/hyper2/pruning/search.go index fbafcb023..08268d17e 100644 --- a/balloon/hyper2/pruning/search.go +++ b/balloon/hyper2/pruning/search.go @@ -1,3 +1,19 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( @@ -6,86 +22,68 @@ import ( "github.com/bbva/qed/balloon/hyper2/navigation" ) -func PruneToFind(index []byte, batches BatchLoader) (Operation, error) { - - var traverse, traverseBatch func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) - var discardBranch func(pos *navigation.Position, batch *BatchNode, iBatch int8) Operation +func PruneToFind(index []byte, batches BatchLoader) *OperationsStack { - traverse = func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) { + var traverse, traverseBatch, discardBranch func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) - var err error + traverse = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { if batch == nil { - batch, err = batches.Load(pos) - if err != nil { - return nil, err - } + batch = batches.Load(pos) } - - return traverseBatch(pos, batch, iBatch) + traverseBatch(pos, batch, iBatch, ops) } - discardBranch = func(pos *navigation.Position, batch *BatchNode, iBatch int8) Operation { + discardBranch = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { if batch.HasElementAt(iBatch) { - return NewUseProvidedOp(pos, batch, iBatch) + ops.PushAll(getProvidedHash(pos, iBatch, batch), collectHash(pos)) + } else { + ops.PushAll(getDefaultHash(pos), collectHash(pos)) } - return NewGetDefaultOp(pos) } - traverseBatch = func(pos *navigation.Position, batch *BatchNode, iBatch int8) (Operation, error) { + traverseBatch = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { // We found a nil value. That means there is no previous node stored on the current // path so we stop traversing because the index does no exist in the tree. - // We return a new shortcut without mutating. if !batch.HasElementAt(iBatch) { - return NewShortcutLeafOp(pos, batch, iBatch, index, nil), nil + ops.Push(noOp(pos)) + return } // at the end of the batch tree if iBatch > 0 && pos.Height%4 == 0 { - op, err := traverse(pos, nil, 0) - if err != nil { - return nil, err - } - return NewLeafOp(pos, batch, iBatch, op), nil + traverse(pos, nil, 0, ops) // load another batch + return } // on an internal node of the subtree // we found a shortcut leaf in our path - if batch.HasElementAt(iBatch) && batch.HasLeafAt(iBatch) { - key, value := batch.GetLeafKVAt(iBatch) - if bytes.Equal(index, key) { - // we found the searched index - return NewShortcutLeafOp(pos, batch, iBatch, key, value), nil - } - // we found another shortcut leaf on our path so the we index - // we are looking for has never been inserted in the tree - return NewShortcutLeafOp(pos, batch, iBatch, key, nil), nil + if batch.HasLeafAt(iBatch) { + // regardless if the key of the shortcut matches the searched index + // we must stop traversing because there are no more leaves below + ops.Push(getProvidedHash(pos, iBatch, batch)) // not collected + return } - var left, right Operation - var err error - rightPos := pos.Right() leftPos := pos.Left() if bytes.Compare(index, rightPos.Index) < 0 { // go to left - left, err = traverse(&leftPos, batch, 2*iBatch+1) - if err != nil { - return nil, err - } - right = NewCollectOp(discardBranch(&rightPos, batch, 2*iBatch+2)) + traverse(pos.Left(), batch, 2*iBatch+1, ops) + discardBranch(rightPos, batch, 2*iBatch+2, ops) } else { // go to right - left = NewCollectOp(discardBranch(&leftPos, batch, 2*iBatch+1)) - right, err = traverse(&rightPos, batch, 2*iBatch+2) - if err != nil { - return nil, err - } + discardBranch(leftPos, batch, 2*iBatch+1, ops) + traverse(rightPos, batch, 2*iBatch+2, ops) } - return NewInnerHashOp(pos, batch, iBatch, left, right), nil - + ops.Push(innerHash(pos)) } + ops := NewOperationsStack() root := navigation.NewRootPosition(uint16(len(index) * 8)) - return traverse(&root, nil, 0) + traverse(root, nil, 0, ops) + if ops.Len() == 0 { + ops.Push(noOp(root)) + } + return ops } diff --git a/balloon/hyper2/pruning/search_test.go b/balloon/hyper2/pruning/search_test.go index ea1cbe7af..6aa0ae503 100644 --- a/balloon/hyper2/pruning/search_test.go +++ b/balloon/hyper2/pruning/search_test.go @@ -1,8 +1,27 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( "testing" + "github.com/bbva/qed/balloon/cache" + "github.com/bbva/qed/balloon/hyper2/navigation" + "github.com/bbva/qed/hashing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,70 +32,273 @@ func TestPruneToFind(t *testing.T) { index []byte cachedBatches map[string][]byte storedBatches map[string][]byte - expectedOp Operation + expectedOps []op }{ { // search for index=0 on an empty tree index: []byte{0}, cachedBatches: map[string][]byte{}, storedBatches: map[string][]byte{}, - expectedOp: shortcut(pos(0, 8), 0, []byte{0x00, 0x00, 0x00, 0x00}, - []byte{0}, nil, - ), + expectedOps: []op{ + {NoOpCode, pos(0, 8)}, // empty audit path + }, }, { // search for index=0 on a tree with one leaf (index=0, value=0) index: []byte{0}, cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {InnerHashCode, pos(0, 8)}, + {CollectHashCode, pos(128, 7)}, + {GetDefaultHashCode, pos(128, 7)}, + {InnerHashCode, pos(0, 7)}, + {CollectHashCode, pos(64, 6)}, + {GetDefaultHashCode, pos(64, 6)}, + {InnerHashCode, pos(0, 6)}, + {CollectHashCode, pos(32, 5)}, + {GetDefaultHashCode, pos(32, 5)}, + {InnerHashCode, pos(0, 5)}, + {CollectHashCode, pos(16, 4)}, + {GetDefaultHashCode, pos(16, 4)}, + {GetProvidedHashCode, pos(0, 4)}, // we stop traversing at the shortcut (index=0) + }, + }, + { + // search for index=1 on tree with 1 leaf (index=0, value=0) + // we traverse until the previous shortcut position even if the leaf does not exist + index: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, }, storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - }, - expectedOp: inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - shortcut(pos(0, 4), 0, []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - []byte{0}, []byte{0}, - ), - ), - collect(getDefault(pos(16, 4))), - ), - collect(getDefault(pos(32, 5))), - ), - collect(getDefault(pos(64, 6))), - ), - collect(getDefault(pos(128, 7))), - ), + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {InnerHashCode, pos(0, 8)}, + {CollectHashCode, pos(128, 7)}, + {GetDefaultHashCode, pos(128, 7)}, + {InnerHashCode, pos(0, 7)}, + {CollectHashCode, pos(64, 6)}, + {GetDefaultHashCode, pos(64, 6)}, + {InnerHashCode, pos(0, 6)}, + {CollectHashCode, pos(32, 5)}, + {GetDefaultHashCode, pos(32, 5)}, + {InnerHashCode, pos(0, 5)}, + {CollectHashCode, pos(16, 4)}, + {GetDefaultHashCode, pos(16, 4)}, + {GetProvidedHashCode, pos(0, 4)}, // stop at the position of the shorcut (index=0) + }, }, { - // search for key=1 on tree with 1 leaf (index: 0, value: 0) + // search for index=1 on tree with 2 leaves ([index=0, value=0], [index=1, value=1]) + // we traverse until the end of the tree index: []byte{1}, cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x01, 0x00, // iBatch 0 -> hash=0x01 + 0x01, 0x00, // iBatch 1 -> hash=0x01 + 0x01, 0x00, // iBatch 3 -> hash=0x01 + 0x01, 0x00, // iBatch 7 -> hash=0x01 + 0x01, 0x00, // iBatch 15 -> hash=0x01 + }, }, storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - }, - expectedOp: inner(pos(0, 8), 0, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 7), 1, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 6), 3, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - inner(pos(0, 5), 7, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - leaf(pos(0, 4), 15, []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - shortcut(pos(0, 4), 0, []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - []byte{0}, nil, - ), - ), - collect(getDefault(pos(16, 4))), - ), - collect(getDefault(pos(32, 5))), - ), - collect(getDefault(pos(64, 6))), - ), - collect(getDefault(pos(128, 7))), - ), + pos(1, 0).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x01, 0x01, // iBatch 0 -> hash=0x01 (shortcut index=0) + 0x01, 0x02, // iBatch 1 -> key=0x01 + 0x01, 0x02, // iBatch 2 -> value=0x01 + }, + pos(0, 0).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + pos(0, 4).StringId(): []byte{ + 0xd1, 0x01, 0x80, 0x00, // bitmap: 11010001 00000001 10000000 00000000 + 0x01, 0x00, // iBatch 0 -> hash=0x01 + 0x01, 0x00, // iBatch 1 -> hash=0x01 + 0x01, 0x00, // iBatch 3 -> hash=0x01 + 0x01, 0x00, // iBatch 7 -> hash=0x01 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + 0x01, 0x00, // iBatch 16 -> hash=0x01 + }, + }, + expectedOps: []op{ + {InnerHashCode, pos(0, 8)}, + {CollectHashCode, pos(128, 7)}, + {GetDefaultHashCode, pos(128, 7)}, + {InnerHashCode, pos(0, 7)}, + {CollectHashCode, pos(64, 6)}, + {GetDefaultHashCode, pos(64, 6)}, + {InnerHashCode, pos(0, 6)}, + {CollectHashCode, pos(32, 5)}, + {GetDefaultHashCode, pos(32, 5)}, + {InnerHashCode, pos(0, 5)}, + {CollectHashCode, pos(16, 4)}, + {GetDefaultHashCode, pos(16, 4)}, + {InnerHashCode, pos(0, 4)}, + {CollectHashCode, pos(8, 3)}, + {GetDefaultHashCode, pos(8, 3)}, + {InnerHashCode, pos(0, 3)}, + {CollectHashCode, pos(4, 2)}, + {GetDefaultHashCode, pos(4, 2)}, + {InnerHashCode, pos(0, 2)}, + {CollectHashCode, pos(2, 1)}, + {GetDefaultHashCode, pos(2, 1)}, + {InnerHashCode, pos(0, 1)}, + {GetProvidedHashCode, pos(1, 0)}, // shortcut found but not collected + {CollectHashCode, pos(0, 0)}, + {GetProvidedHashCode, pos(0, 0)}, // we take the hash of the index=0 position from the batch + }, + }, + { + // search for index=8 on tree with 1 leaf (index: 0, value: 0) + // we traverse until the previous shortcut position even if the leaf does not exist + index: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {InnerHashCode, pos(0, 8)}, + {CollectHashCode, pos(128, 7)}, + {GetDefaultHashCode, pos(128, 7)}, + {InnerHashCode, pos(0, 7)}, + {CollectHashCode, pos(64, 6)}, + {GetDefaultHashCode, pos(64, 6)}, + {InnerHashCode, pos(0, 6)}, + {CollectHashCode, pos(32, 5)}, + {GetDefaultHashCode, pos(32, 5)}, + {InnerHashCode, pos(0, 5)}, + {CollectHashCode, pos(16, 4)}, + {GetDefaultHashCode, pos(16, 4)}, + {GetProvidedHashCode, pos(0, 4)}, // stop at the position of the shorcut (index=0) + }, + }, + { + // search for index=12 on tree with 3 leaves ([index:0, value:0], [index:8, value:8], [index:12, value:12]) + index: []byte{12}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x04, 0x00, // iBatch 0 -> hash=0x04 + 0x04, 0x00, // iBatch 1 -> hash=0x04 + 0x04, 0x00, // iBatch 3 -> hash=0x04 + 0x04, 0x00, // iBatch 7 -> hash=0x04 + 0x04, 0x00, // iBatch 15 -> hash=0x04 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xfe, 0x1e, 0x00, 0x00, // bitmap: 11111110 00011110 00000000 00000000 + 0x04, 0x00, // iBatch 0 -> hash=0x08 + 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) + 0x04, 0x00, // iBatch 2 -> hash=0x04 + 0x00, 0x02, // iBatch 3 -> key=0x00 + 0x00, 0x02, // iBatch 4 -> value=0x00 + 0x08, 0x01, // iBatch 5 -> hash=0x08 (shortcut index=8) + 0x0c, 0x01, // iBatch 6 -> hash=0x0c (shortcut index=12) + 0x08, 0x02, // iBatch 11 -> key=0x08 + 0x08, 0x02, // iBatch 12 -> value=0x08 + 0x0c, 0x02, // iBatch 13 -> key=0x0c + 0x0c, 0x02, // iBatch 14 -> value=0x0c + }, + }, + expectedOps: []op{ + {InnerHashCode, pos(0, 8)}, + {CollectHashCode, pos(128, 7)}, + {GetDefaultHashCode, pos(128, 7)}, + {InnerHashCode, pos(0, 7)}, + {CollectHashCode, pos(64, 6)}, + {GetDefaultHashCode, pos(64, 6)}, + {InnerHashCode, pos(0, 6)}, + {CollectHashCode, pos(32, 5)}, + {GetDefaultHashCode, pos(32, 5)}, + {InnerHashCode, pos(0, 5)}, + {CollectHashCode, pos(16, 4)}, + {GetDefaultHashCode, pos(16, 4)}, + {InnerHashCode, pos(0, 4)}, + {InnerHashCode, pos(8, 3)}, + {GetProvidedHashCode, pos(12, 2)}, // found shortcut index=12 + {CollectHashCode, pos(8, 2)}, + {GetProvidedHashCode, pos(8, 2)}, // shortcut index=8 + {CollectHashCode, pos(0, 3)}, + {GetProvidedHashCode, pos(0, 3)}, // shortcut index=0 + }, + }, + { + // search for index=128 on tree with one leaf ([index:0, value:0] + index: []byte{128}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedOps: []op{ + {InnerHashCode, pos(0, 8)}, + {NoOpCode, pos(128, 7)}, // not found + {CollectHashCode, pos(0, 7)}, + {GetProvidedHashCode, pos(0, 7)}, // we discard the previous path updated by the previous insertion + }, }, } @@ -85,9 +307,247 @@ func TestPruneToFind(t *testing.T) { for i, c := range testCases { loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - prunedOp, err := PruneToFind(c.index, loader) - require.NoError(t, err) - assert.Equalf(t, c.expectedOp, prunedOp, "The pruned operation should match for test case %d", i) + prunedOps := PruneToFind(c.index, loader).List() + require.Truef(t, len(c.expectedOps) == len(prunedOps), "The size of the pruned ops should match the expected for test case %d", i) + for j := 0; j < len(prunedOps); j++ { + assert.Equalf(t, c.expectedOps[j].Code, prunedOps[j].Code, "The pruned operation's code should match for test case %d", i) + assert.Equalf(t, c.expectedOps[j].Pos, prunedOps[j].Pos, "The pruned operation's position should match for test case %d", i) + } } +} + +func TestSearchInterpretation(t *testing.T) { + testCases := []struct { + index []byte + cachedBatches map[string][]byte + storedBatches map[string][]byte + expectedAuditPath navigation.AuditPath + }{ + { + // search for index=0 on empty tree + index: []byte{0}, + cachedBatches: map[string][]byte{}, + storedBatches: map[string][]byte{}, + expectedAuditPath: navigation.AuditPath{}, + }, + { + // search for index=0 on tree with only one leaf + index: []byte{0}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedAuditPath: navigation.AuditPath{ + pos(128, 7).StringId(): []byte{0x0}, + pos(64, 6).StringId(): []byte{0x0}, + pos(32, 5).StringId(): []byte{0x0}, + pos(16, 4).StringId(): []byte{0x0}, + }, + }, + { + // search for index=1 on tree with 1 leaf (index=0, value=0) + // we traverse until the previous shortcut position even if the leaf does not exist + index: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedAuditPath: navigation.AuditPath{ + pos(128, 7).StringId(): []byte{0x0}, + pos(64, 6).StringId(): []byte{0x0}, + pos(32, 5).StringId(): []byte{0x0}, + pos(16, 4).StringId(): []byte{0x0}, + }, + }, + { + // search for index=1 on tree with 2 leaves ([index=0, value=0], [index=1, value=1]) + // we traverse until the end of the tree + index: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x01, 0x00, // iBatch 0 -> hash=0x01 + 0x01, 0x00, // iBatch 1 -> hash=0x01 + 0x01, 0x00, // iBatch 3 -> hash=0x01 + 0x01, 0x00, // iBatch 7 -> hash=0x01 + 0x01, 0x00, // iBatch 15 -> hash=0x01 + }, + }, + storedBatches: map[string][]byte{ + pos(1, 0).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x01, 0x01, // iBatch 0 -> hash=0x01 (shortcut index=0) + 0x01, 0x02, // iBatch 1 -> key=0x01 + 0x01, 0x02, // iBatch 2 -> value=0x01 + }, + pos(0, 0).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + pos(0, 4).StringId(): []byte{ + 0xd1, 0x01, 0x80, 0x00, // bitmap: 11010001 00000001 10000000 00000000 + 0x01, 0x00, // iBatch 0 -> hash=0x01 + 0x01, 0x00, // iBatch 1 -> hash=0x01 + 0x01, 0x00, // iBatch 3 -> hash=0x01 + 0x01, 0x00, // iBatch 7 -> hash=0x01 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + 0x01, 0x00, // iBatch 16 -> hash=0x01 + }, + }, + expectedAuditPath: navigation.AuditPath{ + pos(128, 7).StringId(): []byte{0x0}, + pos(64, 6).StringId(): []byte{0x0}, + pos(32, 5).StringId(): []byte{0x0}, + pos(16, 4).StringId(): []byte{0x0}, + pos(8, 3).StringId(): []byte{0x0}, + pos(4, 2).StringId(): []byte{0x0}, + pos(2, 1).StringId(): []byte{0x0}, + pos(0, 0).StringId(): []byte{0x0}, + }, + }, + { + // search for index=8 on tree with 1 leaf (index: 0, value: 0) + // we traverse until the previous shortcut position even if the leaf does not exist + index: []byte{1}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedAuditPath: navigation.AuditPath{ + pos(128, 7).StringId(): []byte{0x0}, + pos(64, 6).StringId(): []byte{0x0}, + pos(32, 5).StringId(): []byte{0x0}, + pos(16, 4).StringId(): []byte{0x0}, + }, + }, + { + // search for index=12 on tree with 2 leaves ([index:0, value:0], [index:8, value:8]) + index: []byte{12}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x04, 0x00, // iBatch 0 -> hash=0x04 + 0x04, 0x00, // iBatch 1 -> hash=0x04 + 0x04, 0x00, // iBatch 3 -> hash=0x04 + 0x04, 0x00, // iBatch 7 -> hash=0x04 + 0x04, 0x00, // iBatch 15 -> hash=0x04 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xfe, 0x1e, 0x00, 0x00, // bitmap: 11111110 00011110 00000000 00000000 + 0x04, 0x00, // iBatch 0 -> hash=0x08 + 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) + 0x04, 0x00, // iBatch 2 -> hash=0x04 + 0x00, 0x02, // iBatch 3 -> key=0x00 + 0x00, 0x02, // iBatch 4 -> value=0x00 + 0x08, 0x01, // iBatch 5 -> hash=0x08 (shortcut index=8) + 0x0c, 0x01, // iBatch 6 -> hash=0x0c (shortcut index=12) + 0x08, 0x02, // iBatch 11 -> key=0x08 + 0x08, 0x02, // iBatch 12 -> value=0x08 + 0x0c, 0x02, // iBatch 13 -> key=0x0c + 0x0c, 0x02, // iBatch 14 -> value=0x0c + }, + }, + expectedAuditPath: navigation.AuditPath{ + pos(128, 7).StringId(): []byte{0x0}, + pos(64, 6).StringId(): []byte{0x0}, + pos(32, 5).StringId(): []byte{0x0}, + pos(16, 4).StringId(): []byte{0x0}, + pos(8, 2).StringId(): []byte{0x8}, + pos(0, 3).StringId(): []byte{0x0}, + }, + }, + { + // search for index=128 on tree with one leaf ([index:0, value:0] + index: []byte{128}, + cachedBatches: map[string][]byte{ + pos(0, 8).StringId(): []byte{ + 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 + 0x00, 0x00, // iBatch 0 -> hash=0x00 + 0x00, 0x00, // iBatch 1 -> hash=0x00 + 0x00, 0x00, // iBatch 3 -> hash=0x00 + 0x00, 0x00, // iBatch 7 -> hash=0x00 + 0x00, 0x00, // iBatch 15 -> hash=0x00 + }, + }, + storedBatches: map[string][]byte{ + pos(0, 4).StringId(): []byte{ + 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 + 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) + 0x00, 0x02, // iBatch 1 -> key=0x00 + 0x00, 0x02, // iBatch 2 -> value=0x00 + }, + }, + expectedAuditPath: navigation.AuditPath{ + pos(0, 7).StringId(): []byte{0x0}, + }, + }, + } + + batchLevels := uint16(1) + cacheHeightLimit := batchLevels * 4 + defaultHashes := []hashing.Digest{{0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}} + + for i, c := range testCases { + cache := cache.NewFakeCache([]byte{0x0}) + batches := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) + + ops := PruneToFind(c.index, batches) + ctx := &Context{ + Hasher: hashing.NewFakeXorHasher(), + Cache: cache, + DefaultHashes: defaultHashes, + AuditPath: navigation.NewAuditPath(), + } + + ops.Pop().Interpret(ops, ctx) + assert.Equalf(t, c.expectedAuditPath, ctx.AuditPath, "Audit path error in test case %d", i) + + } } diff --git a/balloon/hyper2/pruning2/stack.go b/balloon/hyper2/pruning/stack.go similarity index 98% rename from balloon/hyper2/pruning2/stack.go rename to balloon/hyper2/pruning/stack.go index b36971fbd..cd350c4c7 100644 --- a/balloon/hyper2/pruning2/stack.go +++ b/balloon/hyper2/pruning/stack.go @@ -14,7 +14,7 @@ limitations under the License. */ -package pruning2 +package pruning type OperationsStack []*Operation diff --git a/balloon/hyper2/pruning/test_util.go b/balloon/hyper2/pruning/test_util.go index 94b593105..7d4447b6a 100644 --- a/balloon/hyper2/pruning/test_util.go +++ b/balloon/hyper2/pruning/test_util.go @@ -20,41 +20,13 @@ import ( "github.com/bbva/qed/balloon/hyper2/navigation" ) -func pos(index byte, height uint16) *navigation.Position { - p := navigation.NewPosition([]byte{index}, height) - return &p +func pos(index byte, height uint16) navigation.Position { + return navigation.NewPosition([]byte{index}, height) } -func inner(pos *navigation.Position, iBatch int8, batch []byte, left, right Operation) *InnerHashOp { - return NewInnerHashOp(pos, ParseBatchNode(1, batch), iBatch, left, right) -} - -func leaf(pos *navigation.Position, iBatch int8, batch []byte, op Operation) *LeafOp { - return NewLeafOp(pos, ParseBatchNode(1, batch), iBatch, op) -} - -func shortcut(pos *navigation.Position, iBatch int8, batch []byte, key, value []byte) *ShortcutLeafOp { - return NewShortcutLeafOp(pos, ParseBatchNode(1, batch), iBatch, key, value) -} - -func getDefault(pos *navigation.Position) *GetDefaultOp { - return NewGetDefaultOp(pos) -} - -func useProvided(pos *navigation.Position, iBatch int8, batch []byte) *UseProvidedOp { - return NewUseProvidedOp(pos, ParseBatchNode(1, batch), iBatch) -} - -func putBatch(op Operation, batch []byte) *PutBatchOp { - return NewPutBatchOp(op, ParseBatchNode(1, batch)) -} - -func mutate(op Operation, batch []byte) *MutateBatchOp { - return NewMutateBatchOp(op, ParseBatchNode(1, batch)) -} - -func collect(op Operation) *CollectOp { - return NewCollectOp(op) +type op struct { + Code OperationCode + Pos navigation.Position } type FakeBatchLoader struct { @@ -78,17 +50,17 @@ func NewFakeBatchLoader(cached map[string][]byte, stored map[string][]byte, cach return loader } -func (l *FakeBatchLoader) Load(pos *navigation.Position) (*BatchNode, error) { +func (l *FakeBatchLoader) Load(pos navigation.Position) *BatchNode { if pos.Height > l.cacheHeightLimit { batch, ok := l.cached[pos.StringId()] - if ok { - return batch, nil + if !ok { + return NewEmptyBatchNode(len(pos.Index)) } - return NewEmptyBatchNode(len(pos.Index)), nil + return batch } batch, ok := l.stored[pos.StringId()] - if ok { - return batch, nil + if !ok { + return NewEmptyBatchNode(len(pos.Index)) } - return NewEmptyBatchNode(len(pos.Index)), nil + return batch } diff --git a/balloon/hyper2/pruning/verify.go b/balloon/hyper2/pruning/verify.go index bb7688dfd..c1522a370 100644 --- a/balloon/hyper2/pruning/verify.go +++ b/balloon/hyper2/pruning/verify.go @@ -1,3 +1,19 @@ +/* + Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package pruning import ( @@ -6,33 +22,32 @@ import ( "github.com/bbva/qed/balloon/hyper2/navigation" ) -func PruneToVerify(key, value []byte, auditPathHeight uint16) Operation { +func PruneToVerify(index, value []byte, auditPathHeight uint16) *OperationsStack { - var traverse func(pos *navigation.Position) Operation + var traverse func(pos navigation.Position, ops *OperationsStack) - traverse = func(pos *navigation.Position) Operation { + traverse = func(pos navigation.Position, ops *OperationsStack) { if pos.Height <= auditPathHeight { - // we are at the leaf height - return NewShortcutLeafOp(pos, nil, 0, key, value) + ops.Push(leafHash(pos, value)) + return } - var left, right Operation rightPos := pos.Right() - leftPos := pos.Left() - if bytes.Compare(key, rightPos.Index) < 0 { // go to left - left = traverse(&leftPos) - right = NewGetDefaultOp(&rightPos) + if bytes.Compare(index, rightPos.Index) < 0 { // go to left + traverse(pos.Left(), ops) + ops.Push(getFromPath(rightPos)) } else { // go to right - left = NewGetDefaultOp(&leftPos) - right = traverse(&rightPos) + ops.Push(getFromPath(pos.Left())) + traverse(rightPos, ops) } - return NewInnerHashOp(pos, nil, 0, left, right) + ops.Push(innerHash(pos)) } - root := navigation.NewRootPosition(uint16(len(key) * 8)) - return traverse(&root) + ops := NewOperationsStack() + traverse(navigation.NewRootPosition(uint16(len(index)*8)), ops) + return ops } diff --git a/balloon/hyper2/pruning2/verify_test.go b/balloon/hyper2/pruning/verify_test.go similarity index 99% rename from balloon/hyper2/pruning2/verify_test.go rename to balloon/hyper2/pruning/verify_test.go index f69b92e4a..ba53ddd7a 100644 --- a/balloon/hyper2/pruning2/verify_test.go +++ b/balloon/hyper2/pruning/verify_test.go @@ -14,7 +14,7 @@ limitations under the License. */ -package pruning2 +package pruning import ( "testing" diff --git a/balloon/hyper2/pruning/visitor.go b/balloon/hyper2/pruning/visitor.go deleted file mode 100644 index 5632917e1..000000000 --- a/balloon/hyper2/pruning/visitor.go +++ /dev/null @@ -1,136 +0,0 @@ -package pruning - -import ( - "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/hashing" - "github.com/bbva/qed/storage" -) - -type InsertVisitor struct { - cache cache.ModifiableCache - defaultHashes []hashing.Digest - hasher hashing.Hasher - - mutations []*storage.Mutation -} - -func NewInsertVisitor(hasher hashing.Hasher, cache cache.ModifiableCache, defaultHashes []hashing.Digest) *InsertVisitor { - return &InsertVisitor{ - cache: cache, - defaultHashes: defaultHashes, - hasher: hasher, - mutations: make([]*storage.Mutation, 0), - } -} - -func (v InsertVisitor) Result() []*storage.Mutation { - return v.mutations -} - -func (v *InsertVisitor) VisitShortcutLeafOp(op ShortcutLeafOp) hashing.Digest { - hash := v.hasher.Salted(op.Position().Bytes(), op.Value) - // fmt.Printf("Shortcut hash(%v): %x\n", op.Position(), hash) - op.Batch.AddLeafAt(op.Idx, hash, op.Key, op.Value) - return hash -} - -func (v *InsertVisitor) VisitLeafOp(op LeafOp) hashing.Digest { - hash := op.Operation.Accept(v) - // fmt.Printf("Leaf hash(%v): %x\n", op.Position(), hash) - op.Batch.AddHashAt(op.Idx, hash) - return hash -} - -func (v *InsertVisitor) VisitInnerHashOp(op InnerHashOp) hashing.Digest { - leftHash := op.Left.Accept(v) - rightHash := op.Right.Accept(v) - hash := v.hasher.Salted(op.Position().Bytes(), leftHash, rightHash) - // fmt.Printf("Inner hash(%v): %x, %x => %x\n", op.Position(), leftHash, rightHash, hash) - op.Batch.AddHashAt(op.Idx, hash) - return hash -} - -func (v *InsertVisitor) VisitGetDefaultOp(op GetDefaultOp) hashing.Digest { - hash := v.defaultHashes[op.Position().Height] - // fmt.Printf("Default hash(%v): %x\n", op.Position(), hash) - return hash -} - -func (v *InsertVisitor) VisitUseProvidedOp(op UseProvidedOp) hashing.Digest { - hash := op.Batch.GetElementAt(op.Idx) - // fmt.Printf("Provided hash(%v): %x\n", op.Position(), hash) - return hash -} - -func (v *InsertVisitor) VisitPutBatchOp(op PutBatchOp) hashing.Digest { - hash := op.Operation.Accept(v) - v.cache.Put(op.Position().Bytes(), op.Batch.Serialize()) - // fmt.Printf("Put cache hash(%v) [%d]: %x\n", op.Position(), op.Batch.Serialize(), hash) - return hash -} - -func (v *InsertVisitor) VisitMutateBatchOp(op MutateBatchOp) hashing.Digest { - hash := op.Operation.Accept(v) - v.mutations = append(v.mutations, storage.NewMutation(storage.HyperCachePrefix, op.Position().Bytes(), op.Batch.Serialize())) - // fmt.Printf("Mutate hash(%v) [%d]: %x\n", op.Position(), op.Batch.Serialize(), hash) - return hash -} - -func (v *InsertVisitor) VisitCollectOp(op CollectOp) hashing.Digest { - return op.Operation.Accept(v) -} - -type AuditPathVisitor struct { - hasher hashing.Hasher - defaultHashes []hashing.Digest - auditPath navigation.AuditPath -} - -func NewAuditPathVisitor(hasher hashing.Hasher, defaultHashes []hashing.Digest) *AuditPathVisitor { - return &AuditPathVisitor{ - hasher: hasher, - defaultHashes: defaultHashes, - auditPath: navigation.NewAuditPath(), - } -} - -func (v AuditPathVisitor) Result() navigation.AuditPath { - return v.auditPath -} - -func (v *AuditPathVisitor) VisitShortcutLeafOp(op ShortcutLeafOp) hashing.Digest { - return nil -} - -func (v *AuditPathVisitor) VisitLeafOp(op LeafOp) hashing.Digest { - return op.Operation.Accept(v) -} - -func (v *AuditPathVisitor) VisitInnerHashOp(op InnerHashOp) hashing.Digest { - op.Left.Accept(v) - op.Right.Accept(v) - return nil -} - -func (v *AuditPathVisitor) VisitGetDefaultOp(op GetDefaultOp) hashing.Digest { - return v.defaultHashes[op.Position().Height] -} - -func (v *AuditPathVisitor) VisitUseProvidedOp(op UseProvidedOp) hashing.Digest { - return op.Batch.GetElementAt(op.Idx) -} - -func (v *AuditPathVisitor) VisitPutBatchOp(op PutBatchOp) hashing.Digest { - return op.Operation.Accept(v) -} - -func (v *AuditPathVisitor) VisitMutateBatchOp(op MutateBatchOp) hashing.Digest { - return op.Operation.Accept(v) -} - -func (v *AuditPathVisitor) VisitCollectOp(op CollectOp) hashing.Digest { - hash := op.Operation.Accept(v) - v.auditPath = append(v.auditPath, hash) - return hash -} diff --git a/balloon/hyper2/pruning/visitor_test.go b/balloon/hyper2/pruning/visitor_test.go deleted file mode 100644 index 5ca34faa9..000000000 --- a/balloon/hyper2/pruning/visitor_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package pruning - -import ( - "testing" - - "github.com/bbva/qed/hashing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/storage" -) - -func TestInsertVisitor(t *testing.T) { - - testCases := []struct { - index, value []byte - cachedBatches map[string][]byte - storedBatches map[string][]byte - expectedMutations []*storage.Mutation - expectedElements []*cachedElement - }{ - { - // insert index = 0 on empty tree - index: []byte{0}, - value: []byte{0}, - cachedBatches: map[string][]byte{}, - storedBatches: map[string][]byte{}, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 4).Bytes(), - Value: []byte{0xe0, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x02}, - }, - }, - expectedElements: []*cachedElement{ - newCachedElement(pos(0, 8), []byte{0xd1, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), - }, - }, - } - - batchLevels := uint16(1) - cacheHeightLimit := batchLevels * 4 - - for i, c := range testCases { - cache := cache.NewFakeCache([]byte{0x0}) - loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - defaultHashes := []hashing.Digest{{0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}} - visitor := NewInsertVisitor(hashing.NewFakeXorHasher(), cache, defaultHashes) - - prunedOp, err := PruneToInsert(c.index, c.value, cacheHeightLimit, loader) - require.NoError(t, err) - - prunedOp.Accept(visitor) - - mutations := visitor.Result() - assert.ElementsMatchf(t, c.expectedMutations, mutations, "Mutation error in test case %d", i) - for _, e := range c.expectedElements { - v, _ := cache.Get(e.Pos.Bytes()) - assert.Equalf(t, e.Value, v, "The cached element %v should be cached in test case %d", e, i) - } - } - -} - -type cachedElement struct { - Pos *navigation.Position - Value []byte -} - -func newCachedElement(pos *navigation.Position, value []byte) *cachedElement { - return &cachedElement{pos, value} -} diff --git a/balloon/hyper2/pruning2/batch.go b/balloon/hyper2/pruning2/batch.go deleted file mode 100644 index 0f23e011f..000000000 --- a/balloon/hyper2/pruning2/batch.go +++ /dev/null @@ -1,120 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "fmt" - "strings" - - "github.com/bbva/qed/hashing" -) - -type BatchNode struct { - Batch [][]byte - nodeSize int // in bytes -} - -func NewEmptyBatchNode(nodeSize int) *BatchNode { - return &BatchNode{ - nodeSize: nodeSize, - Batch: make([][]byte, 31, 31), - } -} - -func NewBatchNode(nodeSize int, batch [][]byte) *BatchNode { - return &BatchNode{ - nodeSize: nodeSize, - Batch: batch, - } -} - -func (b BatchNode) String() string { - var strs []string - for i, n := range b.Batch { - strs = append(strs, fmt.Sprintf("[%d - %#x]", i, n)) - } - return strings.Join(strs, "\n") -} - -func (b BatchNode) HasLeafAt(i int8) bool { - return len(b.Batch[i]) > 0 && b.Batch[i][b.nodeSize] == byte(1) -} - -func (b BatchNode) AddHashAt(i int8, value []byte) { - b.Batch[i] = append(value, byte(0)) -} - -func (b BatchNode) AddLeafAt(i int8, hash hashing.Digest, key, value []byte) { - b.Batch[i] = append(hash, byte(1)) - b.Batch[2*i+1] = append(key, byte(2)) - b.Batch[2*i+2] = append(value, byte(2)) -} - -func (b BatchNode) GetLeafKVAt(i int8) ([]byte, []byte) { - return b.Batch[2*i+1][:b.nodeSize], b.Batch[2*i+2][:b.nodeSize] -} - -func (b BatchNode) HasElementAt(i int8) bool { - return len(b.Batch[i]) > 0 -} - -func (b BatchNode) GetElementAt(i int8) []byte { - return b.Batch[i][:b.nodeSize] -} - -func (b BatchNode) ResetElementAt(i int8) { - b.Batch[i] = nil -} - -func (b BatchNode) Serialize() []byte { - serialized := make([]byte, 4) - for i := 0; i < 31; i++ { - if len(b.Batch[i]) != 0 { - bitSet(serialized, i) - serialized = append(serialized, b.Batch[i]...) - } - } - return serialized -} - -func ParseBatch(nodeSize int, value []byte) [][]byte { - batch := make([][]byte, 31, 31) // 31 nodes (including the root) - bitmap := value[:4] // the first 4 bytes define the bitmap - size := nodeSize + 1 - - j := 0 - for i := 0; i < 31; i++ { - if bitIsSet(bitmap, i) { - batch[i] = value[4+size*j : 4+size*(j+1)] - j++ - } - } - - return batch -} - -func ParseBatchNode(nodeSize int, value []byte) *BatchNode { - return NewBatchNode(nodeSize, ParseBatch(nodeSize, value)) -} - -func bitIsSet(bits []byte, i int) bool { - return bits[i/8]&(1< 0 - }) - - if index > 0 && bytes.Equal(l[index-1].Index, leaf.Index) { - return l - } - - l = append(l, leaf) - copy(l[index+1:], l[index:]) - l[index] = leaf - return l - -} - -func (l Leaves) Split(index []byte) (left, right Leaves) { - // the smallest index i where l[i].Index >= index - splitIndex := sort.Search(len(l), func(i int) bool { - return bytes.Compare(l[i].Index, index) >= 0 - }) - return l[:splitIndex], l[splitIndex:] -} - -type TraverseBatch func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) - -func PruneToInsert(index []byte, value []byte, cacheHeightLimit uint16, batches BatchLoader) *OperationsStack { - - var traverse, traverseThroughCache, traverseAfterCache TraverseBatch - - traverse = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { - if batch == nil { - batch = batches.Load(pos) - } - if pos.Height > cacheHeightLimit { - traverseThroughCache(pos, leaves, batch, iBatch, ops) - } else { - traverseAfterCache(pos, leaves, batch, iBatch, ops) - } - } - - traverseThroughCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { - - if len(leaves) == 0 { // discarded branch - if batch.HasElementAt(iBatch) { - ops.Push(getProvidedHash(pos, iBatch, batch)) - } else { - ops.Push(getDefaultHash(pos)) - } - return - } - - // at the end of a batch tree - if iBatch > 0 && pos.Height%4 == 0 { - traverse(pos, leaves, nil, 0, ops) - ops.Push(updateBatchNode(pos, iBatch, batch)) - return - } - - // on an internal node of the subtree - - // we found a node in our path and it is a shortcut leaf - if batch.HasLeafAt(iBatch) { - // push down leaf - key, value := batch.GetLeafKVAt(iBatch) - leaves = leaves.InsertSorted(Leaf{key, value}) - batch.ResetElementAt(iBatch) - batch.ResetElementAt(2*iBatch + 1) - batch.ResetElementAt(2*iBatch + 2) - traverseThroughCache(pos, leaves, batch, iBatch, ops) - return - } - - // on an internal node with more than one leaf - - rightPos := pos.Right() - leftLeaves, rightLeaves := leaves.Split(rightPos.Index) - - traverseThroughCache(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops) - traverseThroughCache(rightPos, rightLeaves, batch, 2*iBatch+2, ops) - - ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch)) - if iBatch == 0 { // it's the root of the batch tree - ops.Push(putInCache(pos, batch)) - } - - } - - traverseAfterCache = func(pos navigation.Position, leaves Leaves, batch *BatchNode, iBatch int8, ops *OperationsStack) { - - if len(leaves) == 0 { // discarded branch - if batch.HasElementAt(iBatch) { - ops.Push(getProvidedHash(pos, iBatch, batch)) - } else { - ops.Push(getDefaultHash(pos)) - } - return - } - - // at the end of the main tree - // this is a special case because we have to mutate even if there exists a previous stored leaf (update scenario) - if pos.IsLeaf() { - if len(leaves) != 1 { - panic("Oops, something went wrong. We cannot have more than one leaf at the end of the main tree") - } - // create or update the leaf with a new shortcut - newBatch := NewEmptyBatchNode(len(pos.Index)) - ops.PushAll( - leafHash(pos, leaves[0].Value), - updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value), - mutateBatch(pos, newBatch), - updateBatchNode(pos, iBatch, batch), - ) - return - } - - // at the end of a subtree - if iBatch > 0 && pos.Height%4 == 0 { - if len(leaves) > 1 { - // with more than one leaf to insert -> it's impossible to be a shortcut leaf - traverse(pos, leaves, nil, 0, ops) - ops.Push(updateBatchNode(pos, iBatch, batch)) - return - } - // with only one leaf to insert -> continue traversing - if batch.HasElementAt(iBatch) { - traverse(pos, leaves, nil, 0, ops) - ops.Push(updateBatchNode(pos, iBatch, batch)) - return - } - // nil value (no previous node stored) so create a new shortcut batch - newBatch := NewEmptyBatchNode(len(pos.Index)) - ops.PushAll( - leafHash(pos, leaves[0].Value), - updateBatchShortcut(pos, 0, newBatch, leaves[0].Index, leaves[0].Value), - mutateBatch(pos, newBatch), - updateBatchNode(pos, iBatch, batch), - ) - return - } - - // on an internal node with only one leaf to insert - - if len(leaves) == 1 { - // we found a nil in our path -> create a shortcut leaf - if !batch.HasElementAt(iBatch) { - ops.PushAll( - leafHash(pos, leaves[0].Value), - updateBatchShortcut(pos, iBatch, batch, leaves[0].Index, leaves[0].Value), - ) - if pos.Height%4 == 0 { // at the root or at a leaf of the subtree (not necessary to check iBatch) - ops.Push(mutateBatch(pos, batch)) - } - return - } - - // we found a node in our path and itis a shortcut leaf - if batch.HasLeafAt(iBatch) { - // push down leaf - key, value := batch.GetLeafKVAt(iBatch) - leaves = leaves.InsertSorted(Leaf{key, value}) - batch.ResetElementAt(iBatch) - batch.ResetElementAt(2*iBatch + 1) - batch.ResetElementAt(2*iBatch + 2) - traverseAfterCache(pos, leaves, batch, iBatch, ops) - return - } - } - - // on an internal node with more than one leaf - rightPos := pos.Right() - leftLeaves, rightLeaves := leaves.Split(rightPos.Index) - - traverseAfterCache(pos.Left(), leftLeaves, batch, 2*iBatch+1, ops) - traverseAfterCache(rightPos, rightLeaves, batch, 2*iBatch+2, ops) - - ops.PushAll(innerHash(pos), updateBatchNode(pos, iBatch, batch)) - if iBatch == 0 { // at root node -> mutate batch - ops.Push(mutateBatch(pos, batch)) - } - - } - - ops := NewOperationsStack() - leaves := make(Leaves, 0) - leaves = leaves.InsertSorted(Leaf{index, value}) - traverse(navigation.NewRootPosition(uint16(len(index)*8)), leaves, nil, 0, ops) - return ops -} diff --git a/balloon/hyper2/pruning2/insert_test.go b/balloon/hyper2/pruning2/insert_test.go deleted file mode 100644 index ad4aadba2..000000000 --- a/balloon/hyper2/pruning2/insert_test.go +++ /dev/null @@ -1,686 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "testing" - - "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/hashing" - "github.com/bbva/qed/storage" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPruneToInsert(t *testing.T) { - - testCases := []struct { - index, value []byte - cachedBatches map[string][]byte - storedBatches map[string][]byte - expectedOps []op - }{ - { - // insert index = 0 on empty tree - index: []byte{0}, - value: []byte{0}, - cachedBatches: map[string][]byte{}, - storedBatches: map[string][]byte{}, - expectedOps: []op{ - {PutInCacheCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(0, 8)}, - {InnerHashCode, pos(0, 8)}, - {GetDefaultHashCode, pos(128, 7)}, - {UpdateBatchNodeCode, pos(0, 7)}, - {InnerHashCode, pos(0, 7)}, - {GetDefaultHashCode, pos(64, 6)}, - {UpdateBatchNodeCode, pos(0, 6)}, - {InnerHashCode, pos(0, 6)}, - {GetDefaultHashCode, pos(32, 5)}, - {UpdateBatchNodeCode, pos(0, 5)}, - {InnerHashCode, pos(0, 5)}, - {GetDefaultHashCode, pos(16, 4)}, - {UpdateBatchNodeCode, pos(0, 4)}, - {MutateBatchCode, pos(0, 4)}, - {UpdateBatchShortcutCode, pos(0, 4)}, - {LeafHashCode, pos(0, 4)}, - }, - }, - { - // update index = 0 on tree with only one leaf - index: []byte{0}, - value: []byte{0}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {PutInCacheCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(0, 8)}, - {InnerHashCode, pos(0, 8)}, - {GetDefaultHashCode, pos(128, 7)}, - {UpdateBatchNodeCode, pos(0, 7)}, - {InnerHashCode, pos(0, 7)}, - {GetDefaultHashCode, pos(64, 6)}, - {UpdateBatchNodeCode, pos(0, 6)}, - {InnerHashCode, pos(0, 6)}, - {GetDefaultHashCode, pos(32, 5)}, - {UpdateBatchNodeCode, pos(0, 5)}, - {InnerHashCode, pos(0, 5)}, - {GetDefaultHashCode, pos(16, 4)}, - {UpdateBatchNodeCode, pos(0, 4)}, - {MutateBatchCode, pos(0, 4)}, - {UpdateBatchShortcutCode, pos(0, 4)}, - {LeafHashCode, pos(0, 4)}, - }, - }, - { - // insert index=1 on tree with 1 leaf (index: 0, value: 0) - // it should push down the previous leaf to the last level - index: []byte{1}, - value: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {PutInCacheCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(0, 8)}, - {InnerHashCode, pos(0, 8)}, - {GetDefaultHashCode, pos(128, 7)}, - {UpdateBatchNodeCode, pos(0, 7)}, - {InnerHashCode, pos(0, 7)}, - {GetDefaultHashCode, pos(64, 6)}, - {UpdateBatchNodeCode, pos(0, 6)}, - {InnerHashCode, pos(0, 6)}, - {GetDefaultHashCode, pos(32, 5)}, - {UpdateBatchNodeCode, pos(0, 5)}, - {InnerHashCode, pos(0, 5)}, - {GetDefaultHashCode, pos(16, 4)}, - {UpdateBatchNodeCode, pos(0, 4)}, - {MutateBatchCode, pos(0, 4)}, // reset previous shortcut - {UpdateBatchNodeCode, pos(0, 4)}, - {InnerHashCode, pos(0, 4)}, - {GetDefaultHashCode, pos(8, 3)}, - {UpdateBatchNodeCode, pos(0, 3)}, - {InnerHashCode, pos(0, 3)}, - {GetDefaultHashCode, pos(4, 2)}, - {UpdateBatchNodeCode, pos(0, 2)}, - {InnerHashCode, pos(0, 2)}, - {GetDefaultHashCode, pos(2, 1)}, - {UpdateBatchNodeCode, pos(0, 1)}, - {InnerHashCode, pos(0, 1)}, - {UpdateBatchNodeCode, pos(1, 0)}, - {MutateBatchCode, pos(1, 0)}, // new batch - {UpdateBatchShortcutCode, pos(1, 0)}, - {LeafHashCode, pos(1, 0)}, - {UpdateBatchNodeCode, pos(0, 0)}, - {MutateBatchCode, pos(0, 0)}, // new batch - {UpdateBatchShortcutCode, pos(0, 0)}, - {LeafHashCode, pos(0, 0)}, - }, - }, - { - // insert index=8 on tree with 1 leaf (index: 0, value: 0) - // it should push down the previous leaf to the next subtree - index: []byte{8}, - value: []byte{8}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {PutInCacheCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(0, 8)}, - {InnerHashCode, pos(0, 8)}, - {GetDefaultHashCode, pos(128, 7)}, - {UpdateBatchNodeCode, pos(0, 7)}, - {InnerHashCode, pos(0, 7)}, - {GetDefaultHashCode, pos(64, 6)}, - {UpdateBatchNodeCode, pos(0, 6)}, - {InnerHashCode, pos(0, 6)}, - {GetDefaultHashCode, pos(32, 5)}, - {UpdateBatchNodeCode, pos(0, 5)}, - {InnerHashCode, pos(0, 5)}, - {GetDefaultHashCode, pos(16, 4)}, - {UpdateBatchNodeCode, pos(0, 4)}, - {MutateBatchCode, pos(0, 4)}, // reset previous shortcut - {UpdateBatchNodeCode, pos(0, 4)}, - {InnerHashCode, pos(0, 4)}, - {UpdateBatchShortcutCode, pos(8, 3)}, - {LeafHashCode, pos(8, 3)}, - {UpdateBatchShortcutCode, pos(0, 3)}, - {LeafHashCode, pos(0, 3)}, - }, - }, - { - - // insert index=12 on tree with 2 leaves ([index:0, value:0], [index:8, value:8]) - // it should push down the leaf with index=8 to the next level - index: []byte{12}, - value: []byte{12}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x08, 0x00, // iBatch 0 -> hash=0x08 - 0x08, 0x00, // iBatch 1 -> hash=0x08 - 0x08, 0x00, // iBatch 3 -> hash=0x08 - 0x08, 0x00, // iBatch 7 -> hash=0x08 - 0x08, 0x00, // iBatch 15 -> hash=0x08 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xfe, 0x00, 0x00, 0x00, // bitmap: 11111110 00000000 00000000 00000000 - 0x08, 0x00, // iBatch 0 -> hash=0x08 - 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) - 0x08, 0x01, // iBatch 2 -> hash=0x08 (shortcut index=8) - 0x00, 0x02, // iBatch 3 -> key=0x00 - 0x00, 0x02, // iBatch 4 -> value=0x00 - 0x08, 0x02, // iBatch 5 -> key=0x08 - 0x08, 0x02, // iBatch 6 -> value=0x08 - }, - }, - expectedOps: []op{ - {PutInCacheCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(0, 8)}, - {InnerHashCode, pos(0, 8)}, - {GetDefaultHashCode, pos(128, 7)}, - {UpdateBatchNodeCode, pos(0, 7)}, - {InnerHashCode, pos(0, 7)}, - {GetDefaultHashCode, pos(64, 6)}, - {UpdateBatchNodeCode, pos(0, 6)}, - {InnerHashCode, pos(0, 6)}, - {GetDefaultHashCode, pos(32, 5)}, - {UpdateBatchNodeCode, pos(0, 5)}, - {InnerHashCode, pos(0, 5)}, - {GetDefaultHashCode, pos(16, 4)}, - {UpdateBatchNodeCode, pos(0, 4)}, - {MutateBatchCode, pos(0, 4)}, - {UpdateBatchNodeCode, pos(0, 4)}, - {InnerHashCode, pos(0, 4)}, - {UpdateBatchNodeCode, pos(8, 3)}, - {InnerHashCode, pos(8, 3)}, - {UpdateBatchShortcutCode, pos(12, 2)}, - {LeafHashCode, pos(12, 2)}, - {UpdateBatchShortcutCode, pos(8, 2)}, - {LeafHashCode, pos(8, 2)}, - {GetProvidedHashCode, pos(0, 3)}, - }, - }, - { - // insert index=128 on tree with one leaf ([index:0, value:0] - index: []byte{128}, - value: []byte{128}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {PutInCacheCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(0, 8)}, - {InnerHashCode, pos(0, 8)}, - {UpdateBatchNodeCode, pos(128, 7)}, - {InnerHashCode, pos(128, 7)}, - {GetDefaultHashCode, pos(192, 6)}, - {UpdateBatchNodeCode, pos(128, 6)}, - {InnerHashCode, pos(128, 6)}, - {GetDefaultHashCode, pos(160, 5)}, - {UpdateBatchNodeCode, pos(128, 5)}, - {InnerHashCode, pos(128, 5)}, - {GetDefaultHashCode, pos(144, 4)}, - {UpdateBatchNodeCode, pos(128, 4)}, - {MutateBatchCode, pos(128, 4)}, - {UpdateBatchShortcutCode, pos(128, 4)}, - {LeafHashCode, pos(128, 4)}, - {GetProvidedHashCode, pos(0, 7)}, - }, - }, - } - - batchLevels := uint16(1) - cacheHeightLimit := batchLevels * 4 - - for i, c := range testCases { - loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - prunedOps := PruneToInsert(c.index, c.value, cacheHeightLimit, loader).List() - require.Truef(t, len(c.expectedOps) == len(prunedOps), "The size of the pruned ops should match the expected for test case %d", i) - for j := 0; j < len(prunedOps); j++ { - assert.Equalf(t, c.expectedOps[j].Code, prunedOps[j].Code, "The pruned operation's code should match for test case %d", i) - assert.Equalf(t, c.expectedOps[j].Pos, prunedOps[j].Pos, "The pruned operation's position should match for test case %d", i) - } - } - -} - -func TestInsertInterpretation(t *testing.T) { - - testCases := []struct { - index, value []byte - cachedBatches map[string][]byte - storedBatches map[string][]byte - expectedMutations []*storage.Mutation - expectedElements []*cachedElement - }{ - { - // insert index = 0 on empty tree - index: []byte{0}, - value: []byte{0}, - cachedBatches: map[string][]byte{}, - storedBatches: map[string][]byte{}, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 4).Bytes(), - Value: []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - }, - expectedElements: []*cachedElement{ - { - Pos: pos(0, 8), - Value: []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - }, - }, - { - // update index = 0 on tree with only one leaf - index: []byte{0}, - value: []byte{0}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 4).Bytes(), - Value: []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - }, - expectedElements: []*cachedElement{ - { - Pos: pos(0, 8), - Value: []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - }, - }, - { - // insert index=1 on tree with 1 leaf (index: 0, value: 0) - // it should push down the previous leaf to the last level - index: []byte{1}, - value: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(1, 0).Bytes(), - Value: []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x01, 0x01, // iBatch 0 -> hash=0x01 (shortcut index=0) - 0x01, 0x02, // iBatch 1 -> key=0x01 - 0x01, 0x02, // iBatch 2 -> value=0x01 - }, - }, - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 0).Bytes(), - Value: []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 4).Bytes(), - Value: []byte{ - 0xd1, 0x01, 0x80, 0x00, // bitmap: 11010001 00000001 10000000 00000000 - 0x01, 0x00, // iBatch 0 -> hash=0x01 - 0x01, 0x00, // iBatch 1 -> hash=0x01 - 0x01, 0x00, // iBatch 3 -> hash=0x01 - 0x01, 0x00, // iBatch 7 -> hash=0x01 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - 0x01, 0x00, // iBatch 16 -> hash=0x01 - }, - }, - }, - expectedElements: []*cachedElement{ - { - Pos: pos(0, 8), - Value: []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x01, 0x00, // iBatch 0 -> hash=0x01 - 0x01, 0x00, // iBatch 1 -> hash=0x01 - 0x01, 0x00, // iBatch 3 -> hash=0x01 - 0x01, 0x00, // iBatch 7 -> hash=0x01 - 0x01, 0x00, // iBatch 15 -> hash=0x01 - }, - }, - }, - }, - { - // insert index=8 on tree with 1 leaf (index: 0, value: 0) - // it should push down the previous leaf to the next subtree - index: []byte{8}, - value: []byte{8}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 4).Bytes(), - Value: []byte{ - 0xfe, 0x00, 0x00, 0x00, // bitmap: 11111110 00000000 00000000 00000000 - 0x08, 0x00, // iBatch 0 -> hash=0x08 - 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) - 0x08, 0x01, // iBatch 2 -> hash=0x08 (shortcut index=8) - 0x00, 0x02, // iBatch 3 -> key=0x00 - 0x00, 0x02, // iBatch 4 -> value=0x00 - 0x08, 0x02, // iBatch 5 -> key=0x08 - 0x08, 0x02, // iBatch 6 -> value=0x08 - }, - }, - }, - expectedElements: []*cachedElement{ - { - Pos: pos(0, 8), - Value: []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x08, 0x00, // iBatch 0 -> hash=0x08 - 0x08, 0x00, // iBatch 1 -> hash=0x08 - 0x08, 0x00, // iBatch 3 -> hash=0x08 - 0x08, 0x00, // iBatch 7 -> hash=0x08 - 0x08, 0x00, // iBatch 15 -> hash=0x08 - }, - }, - }, - }, - { - // insert index=12 on tree with 2 leaves ([index:0, value:0], [index:8, value:8]) - // it should push down the leaf with index=8 to the next level - index: []byte{12}, - value: []byte{12}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x08, 0x00, // iBatch 0 -> hash=0x08 - 0x08, 0x00, // iBatch 1 -> hash=0x08 - 0x08, 0x00, // iBatch 3 -> hash=0x08 - 0x08, 0x00, // iBatch 7 -> hash=0x08 - 0x08, 0x00, // iBatch 15 -> hash=0x08 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xfe, 0x00, 0x00, 0x00, // bitmap: 11111110 00000000 00000000 00000000 - 0x08, 0x00, // iBatch 0 -> hash=0x08 - 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) - 0x08, 0x01, // iBatch 2 -> hash=0x08 (shortcut index=8) - 0x00, 0x02, // iBatch 3 -> key=0x00 - 0x00, 0x02, // iBatch 4 -> value=0x00 - 0x08, 0x02, // iBatch 5 -> key=0x08 - 0x08, 0x02, // iBatch 6 -> value=0x08 - }, - }, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(0, 4).Bytes(), - Value: []byte{ - 0xfe, 0x1e, 0x00, 0x00, // bitmap: 11111110 00011110 00000000 00000000 - 0x04, 0x00, // iBatch 0 -> hash=0x08 - 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) - 0x04, 0x00, // iBatch 2 -> hash=0x04 - 0x00, 0x02, // iBatch 3 -> key=0x00 - 0x00, 0x02, // iBatch 4 -> value=0x00 - 0x08, 0x01, // iBatch 5 -> hash=0x08 (shortcut index=8) - 0x0c, 0x01, // iBatch 6 -> hash=0x0c (shortcut index=12) - 0x08, 0x02, // iBatch 11 -> key=0x08 - 0x08, 0x02, // iBatch 12 -> value=0x08 - 0x0c, 0x02, // iBatch 13 -> key=0x0c - 0x0c, 0x02, // iBatch 14 -> value=0x0c - }, - }, - }, - expectedElements: []*cachedElement{ - { - Pos: pos(0, 8), - Value: []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x04, 0x00, // iBatch 0 -> hash=0x04 - 0x04, 0x00, // iBatch 1 -> hash=0x04 - 0x04, 0x00, // iBatch 3 -> hash=0x04 - 0x04, 0x00, // iBatch 7 -> hash=0x04 - 0x04, 0x00, // iBatch 15 -> hash=0x04 - }, - }, - }, - }, - { - // insert index=128 on tree with one leaf ([index:0, value:0] - index: []byte{128}, - value: []byte{128}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedMutations: []*storage.Mutation{ - { - Prefix: storage.HyperCachePrefix, - Key: pos(128, 4).Bytes(), - Value: []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x80, 0x01, // iBatch 0 -> hash=0x80 (shortcut index=128) - 0x80, 0x02, // iBatch 1 -> key=0x80 - 0x80, 0x02, // iBatch 2 -> value=0x80 - }, - }, - }, - expectedElements: []*cachedElement{ - { - Pos: pos(0, 8), - Value: []byte{ - 0xf5, 0x11, 0x01, 0x00, // bitmap: 11110101 00010001 00000001 00000000 - 0x80, 0x00, // iBatch 0 -> hash=0x80 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x80, 0x00, // iBatch 2 -> hash=0x80 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x80, 0x00, // iBatch 5 -> hash=0x80 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x80, 0x00, // iBatch 11 -> hash=0x80 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - 0x80, 0x00, // iBatch 23 -> hash=0x80 - }, - }, - }, - }, - } - - batchLevels := uint16(1) - cacheHeightLimit := batchLevels * 4 - defaultHashes := []hashing.Digest{{0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}} - - for i, c := range testCases { - cache := cache.NewFakeCache([]byte{0x0}) - batches := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - - ops := PruneToInsert(c.index, c.value, cacheHeightLimit, batches) - ctx := &Context{ - Hasher: hashing.NewFakeXorHasher(), - Cache: cache, - DefaultHashes: defaultHashes, - Mutations: make([]*storage.Mutation, 0), - } - - ops.Pop().Interpret(ops, ctx) - - assert.ElementsMatchf(t, c.expectedMutations, ctx.Mutations, "Mutation error in test case %d", i) - for _, e := range c.expectedElements { - v, _ := cache.Get(e.Pos.Bytes()) - assert.Equalf(t, e.Value, v, "The cached element %v should be cached in test case %d", e, i) - } - } - -} - -type cachedElement struct { - Pos navigation.Position - Value []byte -} diff --git a/balloon/hyper2/pruning2/loader.go b/balloon/hyper2/pruning2/loader.go deleted file mode 100644 index 98343a38a..000000000 --- a/balloon/hyper2/pruning2/loader.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "github.com/bbva/qed/log" - - "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/storage" -) - -type BatchLoader interface { - Load(pos navigation.Position) *BatchNode -} - -// TODO maybe use a function -type DefaultBatchLoader struct { - cacheHeightLimit uint16 - cache cache.Cache - store storage.Store -} - -func NewDefaultBatchLoader(store storage.Store, cache cache.Cache, cacheHeightLimit uint16) *DefaultBatchLoader { - return &DefaultBatchLoader{ - cacheHeightLimit: cacheHeightLimit, - cache: cache, - store: store, - } -} - -func (l DefaultBatchLoader) Load(pos navigation.Position) *BatchNode { - if pos.Height > l.cacheHeightLimit { - return l.loadBatchFromCache(pos) - } - return l.loadBatchFromStore(pos) -} - -func (l DefaultBatchLoader) loadBatchFromCache(pos navigation.Position) *BatchNode { - value, ok := l.cache.Get(pos.Bytes()) - if !ok { - return NewEmptyBatchNode(len(pos.Index)) - } - batch := ParseBatchNode(len(pos.Index), value) - return batch -} - -func (l DefaultBatchLoader) loadBatchFromStore(pos navigation.Position) *BatchNode { - kv, err := l.store.Get(storage.HyperCachePrefix, pos.Bytes()) - if err != nil { - if err == storage.ErrKeyNotFound { - return NewEmptyBatchNode(len(pos.Index)) - } - log.Fatalf("Oops, something went wrong. Unable to load batch: %v", err) - } - batch := ParseBatchNode(len(pos.Index), kv.Value) - return batch -} diff --git a/balloon/hyper2/pruning2/operation.go b/balloon/hyper2/pruning2/operation.go deleted file mode 100644 index e2b0f81c4..000000000 --- a/balloon/hyper2/pruning2/operation.go +++ /dev/null @@ -1,183 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/hashing" - "github.com/bbva/qed/log" - "github.com/bbva/qed/storage" -) - -type Context struct { - Hasher hashing.Hasher - Cache cache.ModifiableCache - DefaultHashes []hashing.Digest - Mutations []*storage.Mutation - AuditPath navigation.AuditPath -} - -type OperationCode int - -const ( - LeafHashCode OperationCode = iota - InnerHashCode - UpdateBatchNodeCode - UpdateBatchShortcutCode - GetDefaultHashCode - GetProvidedHashCode - PutInCacheCode - MutateBatchCode - CollectHashCode - GetFromPathCode - NoOpCode -) - -type Interpreter func(ops *OperationsStack, c *Context) hashing.Digest - -type Operation struct { - Code OperationCode - Pos navigation.Position - Interpret Interpreter -} - -func leafHash(pos navigation.Position, value []byte) *Operation { - return &Operation{ - Code: LeafHashCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - return c.Hasher.Salted(pos.Bytes(), value) - }, - } -} - -func innerHash(pos navigation.Position) *Operation { - return &Operation{ - Code: InnerHashCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - leftHash := ops.Pop().Interpret(ops, c) - rightHash := ops.Pop().Interpret(ops, c) - return c.Hasher.Salted(pos.Bytes(), leftHash, rightHash) - }, - } -} - -func updateBatchNode(pos navigation.Position, idx int8, batch *BatchNode) *Operation { - return &Operation{ - Code: UpdateBatchNodeCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - hash := ops.Pop().Interpret(ops, c) - batch.AddHashAt(idx, hash) - return hash - }, - } -} - -func updateBatchShortcut(pos navigation.Position, idx int8, batch *BatchNode, key, value []byte) *Operation { - return &Operation{ - Code: UpdateBatchShortcutCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - hash := ops.Pop().Interpret(ops, c) - batch.AddLeafAt(idx, hash, key, value) - return hash - }, - } -} - -func getDefaultHash(pos navigation.Position) *Operation { - return &Operation{ - Code: GetDefaultHashCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - return c.DefaultHashes[pos.Height] - }, - } -} - -func getProvidedHash(pos navigation.Position, idx int8, batch *BatchNode) *Operation { - return &Operation{ - Code: GetProvidedHashCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - return batch.GetElementAt(idx) - }, - } -} - -func putInCache(pos navigation.Position, batch *BatchNode) *Operation { - return &Operation{ - Code: PutInCacheCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - hash := ops.Pop().Interpret(ops, c) - c.Cache.Put(pos.Bytes(), batch.Serialize()) - return hash - }, - } -} - -func mutateBatch(pos navigation.Position, batch *BatchNode) *Operation { - return &Operation{ - Code: MutateBatchCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - hash := ops.Pop().Interpret(ops, c) - c.Mutations = append(c.Mutations, storage.NewMutation(storage.HyperCachePrefix, pos.Bytes(), batch.Serialize())) - return hash - }, - } -} - -func collectHash(pos navigation.Position) *Operation { - return &Operation{ - Code: CollectHashCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - hash := ops.Pop().Interpret(ops, c) - c.AuditPath[pos.StringId()] = hash - return hash - }, - } -} - -func getFromPath(pos navigation.Position) *Operation { - return &Operation{ - Code: GetFromPathCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - hash, ok := c.AuditPath.Get(pos) - if !ok { - log.Fatalf("Oops, something went wrong. Invalid position in audit path") - } - return hash - }, - } -} - -func noOp(pos navigation.Position) *Operation { - return &Operation{ - Code: NoOpCode, - Pos: pos, - Interpret: func(ops *OperationsStack, c *Context) hashing.Digest { - return nil - }, - } -} diff --git a/balloon/hyper2/pruning2/search.go b/balloon/hyper2/pruning2/search.go deleted file mode 100644 index 133ba6d51..000000000 --- a/balloon/hyper2/pruning2/search.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "bytes" - - "github.com/bbva/qed/balloon/hyper2/navigation" -) - -func PruneToFind(index []byte, batches BatchLoader) *OperationsStack { - - var traverse, traverseBatch, discardBranch func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) - - traverse = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { - if batch == nil { - batch = batches.Load(pos) - } - traverseBatch(pos, batch, iBatch, ops) - } - - discardBranch = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { - if batch.HasElementAt(iBatch) { - ops.PushAll(getProvidedHash(pos, iBatch, batch), collectHash(pos)) - } else { - ops.PushAll(getDefaultHash(pos), collectHash(pos)) - } - } - - traverseBatch = func(pos navigation.Position, batch *BatchNode, iBatch int8, ops *OperationsStack) { - - // We found a nil value. That means there is no previous node stored on the current - // path so we stop traversing because the index does no exist in the tree. - if !batch.HasElementAt(iBatch) { - ops.Push(noOp(pos)) - return - } - - // at the end of the batch tree - if iBatch > 0 && pos.Height%4 == 0 { - traverse(pos, nil, 0, ops) // load another batch - return - } - - // on an internal node of the subtree - - // we found a shortcut leaf in our path - if batch.HasLeafAt(iBatch) { - // regardless if the key of the shortcut matches the searched index - // we must stop traversing because there are no more leaves below - ops.Push(getProvidedHash(pos, iBatch, batch)) // not collected - return - } - - rightPos := pos.Right() - leftPos := pos.Left() - if bytes.Compare(index, rightPos.Index) < 0 { // go to left - traverse(pos.Left(), batch, 2*iBatch+1, ops) - discardBranch(rightPos, batch, 2*iBatch+2, ops) - } else { // go to right - discardBranch(leftPos, batch, 2*iBatch+1, ops) - traverse(rightPos, batch, 2*iBatch+2, ops) - } - - ops.Push(innerHash(pos)) - } - - ops := NewOperationsStack() - root := navigation.NewRootPosition(uint16(len(index) * 8)) - traverse(root, nil, 0, ops) - if ops.Len() == 0 { - ops.Push(noOp(root)) - } - return ops -} diff --git a/balloon/hyper2/pruning2/search_test.go b/balloon/hyper2/pruning2/search_test.go deleted file mode 100644 index 42bd51742..000000000 --- a/balloon/hyper2/pruning2/search_test.go +++ /dev/null @@ -1,553 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "testing" - - "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/navigation" - "github.com/bbva/qed/hashing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPruneToFind(t *testing.T) { - - testCases := []struct { - index []byte - cachedBatches map[string][]byte - storedBatches map[string][]byte - expectedOps []op - }{ - { - // search for index=0 on an empty tree - index: []byte{0}, - cachedBatches: map[string][]byte{}, - storedBatches: map[string][]byte{}, - expectedOps: []op{ - {NoOpCode, pos(0, 8)}, // empty audit path - }, - }, - { - // search for index=0 on a tree with one leaf (index=0, value=0) - index: []byte{0}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {InnerHashCode, pos(0, 8)}, - {CollectHashCode, pos(128, 7)}, - {GetDefaultHashCode, pos(128, 7)}, - {InnerHashCode, pos(0, 7)}, - {CollectHashCode, pos(64, 6)}, - {GetDefaultHashCode, pos(64, 6)}, - {InnerHashCode, pos(0, 6)}, - {CollectHashCode, pos(32, 5)}, - {GetDefaultHashCode, pos(32, 5)}, - {InnerHashCode, pos(0, 5)}, - {CollectHashCode, pos(16, 4)}, - {GetDefaultHashCode, pos(16, 4)}, - {GetProvidedHashCode, pos(0, 4)}, // we stop traversing at the shortcut (index=0) - }, - }, - { - // search for index=1 on tree with 1 leaf (index=0, value=0) - // we traverse until the previous shortcut position even if the leaf does not exist - index: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {InnerHashCode, pos(0, 8)}, - {CollectHashCode, pos(128, 7)}, - {GetDefaultHashCode, pos(128, 7)}, - {InnerHashCode, pos(0, 7)}, - {CollectHashCode, pos(64, 6)}, - {GetDefaultHashCode, pos(64, 6)}, - {InnerHashCode, pos(0, 6)}, - {CollectHashCode, pos(32, 5)}, - {GetDefaultHashCode, pos(32, 5)}, - {InnerHashCode, pos(0, 5)}, - {CollectHashCode, pos(16, 4)}, - {GetDefaultHashCode, pos(16, 4)}, - {GetProvidedHashCode, pos(0, 4)}, // stop at the position of the shorcut (index=0) - }, - }, - { - // search for index=1 on tree with 2 leaves ([index=0, value=0], [index=1, value=1]) - // we traverse until the end of the tree - index: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x01, 0x00, // iBatch 0 -> hash=0x01 - 0x01, 0x00, // iBatch 1 -> hash=0x01 - 0x01, 0x00, // iBatch 3 -> hash=0x01 - 0x01, 0x00, // iBatch 7 -> hash=0x01 - 0x01, 0x00, // iBatch 15 -> hash=0x01 - }, - }, - storedBatches: map[string][]byte{ - pos(1, 0).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x01, 0x01, // iBatch 0 -> hash=0x01 (shortcut index=0) - 0x01, 0x02, // iBatch 1 -> key=0x01 - 0x01, 0x02, // iBatch 2 -> value=0x01 - }, - pos(0, 0).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - pos(0, 4).StringId(): []byte{ - 0xd1, 0x01, 0x80, 0x00, // bitmap: 11010001 00000001 10000000 00000000 - 0x01, 0x00, // iBatch 0 -> hash=0x01 - 0x01, 0x00, // iBatch 1 -> hash=0x01 - 0x01, 0x00, // iBatch 3 -> hash=0x01 - 0x01, 0x00, // iBatch 7 -> hash=0x01 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - 0x01, 0x00, // iBatch 16 -> hash=0x01 - }, - }, - expectedOps: []op{ - {InnerHashCode, pos(0, 8)}, - {CollectHashCode, pos(128, 7)}, - {GetDefaultHashCode, pos(128, 7)}, - {InnerHashCode, pos(0, 7)}, - {CollectHashCode, pos(64, 6)}, - {GetDefaultHashCode, pos(64, 6)}, - {InnerHashCode, pos(0, 6)}, - {CollectHashCode, pos(32, 5)}, - {GetDefaultHashCode, pos(32, 5)}, - {InnerHashCode, pos(0, 5)}, - {CollectHashCode, pos(16, 4)}, - {GetDefaultHashCode, pos(16, 4)}, - {InnerHashCode, pos(0, 4)}, - {CollectHashCode, pos(8, 3)}, - {GetDefaultHashCode, pos(8, 3)}, - {InnerHashCode, pos(0, 3)}, - {CollectHashCode, pos(4, 2)}, - {GetDefaultHashCode, pos(4, 2)}, - {InnerHashCode, pos(0, 2)}, - {CollectHashCode, pos(2, 1)}, - {GetDefaultHashCode, pos(2, 1)}, - {InnerHashCode, pos(0, 1)}, - {GetProvidedHashCode, pos(1, 0)}, // shortcut found but not collected - {CollectHashCode, pos(0, 0)}, - {GetProvidedHashCode, pos(0, 0)}, // we take the hash of the index=0 position from the batch - }, - }, - { - // search for index=8 on tree with 1 leaf (index: 0, value: 0) - // we traverse until the previous shortcut position even if the leaf does not exist - index: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {InnerHashCode, pos(0, 8)}, - {CollectHashCode, pos(128, 7)}, - {GetDefaultHashCode, pos(128, 7)}, - {InnerHashCode, pos(0, 7)}, - {CollectHashCode, pos(64, 6)}, - {GetDefaultHashCode, pos(64, 6)}, - {InnerHashCode, pos(0, 6)}, - {CollectHashCode, pos(32, 5)}, - {GetDefaultHashCode, pos(32, 5)}, - {InnerHashCode, pos(0, 5)}, - {CollectHashCode, pos(16, 4)}, - {GetDefaultHashCode, pos(16, 4)}, - {GetProvidedHashCode, pos(0, 4)}, // stop at the position of the shorcut (index=0) - }, - }, - { - // search for index=12 on tree with 3 leaves ([index:0, value:0], [index:8, value:8], [index:12, value:12]) - index: []byte{12}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x04, 0x00, // iBatch 0 -> hash=0x04 - 0x04, 0x00, // iBatch 1 -> hash=0x04 - 0x04, 0x00, // iBatch 3 -> hash=0x04 - 0x04, 0x00, // iBatch 7 -> hash=0x04 - 0x04, 0x00, // iBatch 15 -> hash=0x04 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xfe, 0x1e, 0x00, 0x00, // bitmap: 11111110 00011110 00000000 00000000 - 0x04, 0x00, // iBatch 0 -> hash=0x08 - 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) - 0x04, 0x00, // iBatch 2 -> hash=0x04 - 0x00, 0x02, // iBatch 3 -> key=0x00 - 0x00, 0x02, // iBatch 4 -> value=0x00 - 0x08, 0x01, // iBatch 5 -> hash=0x08 (shortcut index=8) - 0x0c, 0x01, // iBatch 6 -> hash=0x0c (shortcut index=12) - 0x08, 0x02, // iBatch 11 -> key=0x08 - 0x08, 0x02, // iBatch 12 -> value=0x08 - 0x0c, 0x02, // iBatch 13 -> key=0x0c - 0x0c, 0x02, // iBatch 14 -> value=0x0c - }, - }, - expectedOps: []op{ - {InnerHashCode, pos(0, 8)}, - {CollectHashCode, pos(128, 7)}, - {GetDefaultHashCode, pos(128, 7)}, - {InnerHashCode, pos(0, 7)}, - {CollectHashCode, pos(64, 6)}, - {GetDefaultHashCode, pos(64, 6)}, - {InnerHashCode, pos(0, 6)}, - {CollectHashCode, pos(32, 5)}, - {GetDefaultHashCode, pos(32, 5)}, - {InnerHashCode, pos(0, 5)}, - {CollectHashCode, pos(16, 4)}, - {GetDefaultHashCode, pos(16, 4)}, - {InnerHashCode, pos(0, 4)}, - {InnerHashCode, pos(8, 3)}, - {GetProvidedHashCode, pos(12, 2)}, // found shortcut index=12 - {CollectHashCode, pos(8, 2)}, - {GetProvidedHashCode, pos(8, 2)}, // shortcut index=8 - {CollectHashCode, pos(0, 3)}, - {GetProvidedHashCode, pos(0, 3)}, // shortcut index=0 - }, - }, - { - // search for index=128 on tree with one leaf ([index:0, value:0] - index: []byte{128}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedOps: []op{ - {InnerHashCode, pos(0, 8)}, - {NoOpCode, pos(128, 7)}, // not found - {CollectHashCode, pos(0, 7)}, - {GetProvidedHashCode, pos(0, 7)}, // we discard the previous path updated by the previous insertion - }, - }, - } - - batchLevels := uint16(1) - cacheHeightLimit := batchLevels * 4 - - for i, c := range testCases { - loader := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - prunedOps := PruneToFind(c.index, loader).List() - require.Truef(t, len(c.expectedOps) == len(prunedOps), "The size of the pruned ops should match the expected for test case %d", i) - for j := 0; j < len(prunedOps); j++ { - assert.Equalf(t, c.expectedOps[j].Code, prunedOps[j].Code, "The pruned operation's code should match for test case %d", i) - assert.Equalf(t, c.expectedOps[j].Pos, prunedOps[j].Pos, "The pruned operation's position should match for test case %d", i) - } - } -} - -func TestSearchInterpretation(t *testing.T) { - - testCases := []struct { - index []byte - cachedBatches map[string][]byte - storedBatches map[string][]byte - expectedAuditPath navigation.AuditPath - }{ - { - // search for index=0 on empty tree - index: []byte{0}, - cachedBatches: map[string][]byte{}, - storedBatches: map[string][]byte{}, - expectedAuditPath: navigation.AuditPath{}, - }, - { - // search for index=0 on tree with only one leaf - index: []byte{0}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedAuditPath: navigation.AuditPath{ - pos(128, 7).StringId(): []byte{0x0}, - pos(64, 6).StringId(): []byte{0x0}, - pos(32, 5).StringId(): []byte{0x0}, - pos(16, 4).StringId(): []byte{0x0}, - }, - }, - { - // search for index=1 on tree with 1 leaf (index=0, value=0) - // we traverse until the previous shortcut position even if the leaf does not exist - index: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedAuditPath: navigation.AuditPath{ - pos(128, 7).StringId(): []byte{0x0}, - pos(64, 6).StringId(): []byte{0x0}, - pos(32, 5).StringId(): []byte{0x0}, - pos(16, 4).StringId(): []byte{0x0}, - }, - }, - { - // search for index=1 on tree with 2 leaves ([index=0, value=0], [index=1, value=1]) - // we traverse until the end of the tree - index: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x01, 0x00, // iBatch 0 -> hash=0x01 - 0x01, 0x00, // iBatch 1 -> hash=0x01 - 0x01, 0x00, // iBatch 3 -> hash=0x01 - 0x01, 0x00, // iBatch 7 -> hash=0x01 - 0x01, 0x00, // iBatch 15 -> hash=0x01 - }, - }, - storedBatches: map[string][]byte{ - pos(1, 0).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x01, 0x01, // iBatch 0 -> hash=0x01 (shortcut index=0) - 0x01, 0x02, // iBatch 1 -> key=0x01 - 0x01, 0x02, // iBatch 2 -> value=0x01 - }, - pos(0, 0).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - pos(0, 4).StringId(): []byte{ - 0xd1, 0x01, 0x80, 0x00, // bitmap: 11010001 00000001 10000000 00000000 - 0x01, 0x00, // iBatch 0 -> hash=0x01 - 0x01, 0x00, // iBatch 1 -> hash=0x01 - 0x01, 0x00, // iBatch 3 -> hash=0x01 - 0x01, 0x00, // iBatch 7 -> hash=0x01 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - 0x01, 0x00, // iBatch 16 -> hash=0x01 - }, - }, - expectedAuditPath: navigation.AuditPath{ - pos(128, 7).StringId(): []byte{0x0}, - pos(64, 6).StringId(): []byte{0x0}, - pos(32, 5).StringId(): []byte{0x0}, - pos(16, 4).StringId(): []byte{0x0}, - pos(8, 3).StringId(): []byte{0x0}, - pos(4, 2).StringId(): []byte{0x0}, - pos(2, 1).StringId(): []byte{0x0}, - pos(0, 0).StringId(): []byte{0x0}, - }, - }, - { - // search for index=8 on tree with 1 leaf (index: 0, value: 0) - // we traverse until the previous shortcut position even if the leaf does not exist - index: []byte{1}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedAuditPath: navigation.AuditPath{ - pos(128, 7).StringId(): []byte{0x0}, - pos(64, 6).StringId(): []byte{0x0}, - pos(32, 5).StringId(): []byte{0x0}, - pos(16, 4).StringId(): []byte{0x0}, - }, - }, - { - // search for index=12 on tree with 2 leaves ([index:0, value:0], [index:8, value:8]) - index: []byte{12}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x04, 0x00, // iBatch 0 -> hash=0x04 - 0x04, 0x00, // iBatch 1 -> hash=0x04 - 0x04, 0x00, // iBatch 3 -> hash=0x04 - 0x04, 0x00, // iBatch 7 -> hash=0x04 - 0x04, 0x00, // iBatch 15 -> hash=0x04 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xfe, 0x1e, 0x00, 0x00, // bitmap: 11111110 00011110 00000000 00000000 - 0x04, 0x00, // iBatch 0 -> hash=0x08 - 0x00, 0x01, // iBatch 1 -> hash=0x00 (shortcut index=0) - 0x04, 0x00, // iBatch 2 -> hash=0x04 - 0x00, 0x02, // iBatch 3 -> key=0x00 - 0x00, 0x02, // iBatch 4 -> value=0x00 - 0x08, 0x01, // iBatch 5 -> hash=0x08 (shortcut index=8) - 0x0c, 0x01, // iBatch 6 -> hash=0x0c (shortcut index=12) - 0x08, 0x02, // iBatch 11 -> key=0x08 - 0x08, 0x02, // iBatch 12 -> value=0x08 - 0x0c, 0x02, // iBatch 13 -> key=0x0c - 0x0c, 0x02, // iBatch 14 -> value=0x0c - }, - }, - expectedAuditPath: navigation.AuditPath{ - pos(128, 7).StringId(): []byte{0x0}, - pos(64, 6).StringId(): []byte{0x0}, - pos(32, 5).StringId(): []byte{0x0}, - pos(16, 4).StringId(): []byte{0x0}, - pos(8, 2).StringId(): []byte{0x8}, - pos(0, 3).StringId(): []byte{0x0}, - }, - }, - { - // search for index=128 on tree with one leaf ([index:0, value:0] - index: []byte{128}, - cachedBatches: map[string][]byte{ - pos(0, 8).StringId(): []byte{ - 0xd1, 0x01, 0x00, 0x00, // bitmap: 11010001 00000001 00000000 00000000 - 0x00, 0x00, // iBatch 0 -> hash=0x00 - 0x00, 0x00, // iBatch 1 -> hash=0x00 - 0x00, 0x00, // iBatch 3 -> hash=0x00 - 0x00, 0x00, // iBatch 7 -> hash=0x00 - 0x00, 0x00, // iBatch 15 -> hash=0x00 - }, - }, - storedBatches: map[string][]byte{ - pos(0, 4).StringId(): []byte{ - 0xe0, 0x00, 0x00, 0x00, // bitmap: 11100000 00000000 00000000 00000000 - 0x00, 0x01, // iBatch 0 -> hash=0x00 (shortcut index=0) - 0x00, 0x02, // iBatch 1 -> key=0x00 - 0x00, 0x02, // iBatch 2 -> value=0x00 - }, - }, - expectedAuditPath: navigation.AuditPath{ - pos(0, 7).StringId(): []byte{0x0}, - }, - }, - } - - batchLevels := uint16(1) - cacheHeightLimit := batchLevels * 4 - defaultHashes := []hashing.Digest{{0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}, {0}} - - for i, c := range testCases { - cache := cache.NewFakeCache([]byte{0x0}) - batches := NewFakeBatchLoader(c.cachedBatches, c.storedBatches, cacheHeightLimit) - - ops := PruneToFind(c.index, batches) - ctx := &Context{ - Hasher: hashing.NewFakeXorHasher(), - Cache: cache, - DefaultHashes: defaultHashes, - AuditPath: navigation.NewAuditPath(), - } - - ops.Pop().Interpret(ops, ctx) - assert.Equalf(t, c.expectedAuditPath, ctx.AuditPath, "Audit path error in test case %d", i) - - } -} diff --git a/balloon/hyper2/pruning2/test_util.go b/balloon/hyper2/pruning2/test_util.go deleted file mode 100644 index df025198d..000000000 --- a/balloon/hyper2/pruning2/test_util.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "github.com/bbva/qed/balloon/hyper2/navigation" -) - -func pos(index byte, height uint16) navigation.Position { - return navigation.NewPosition([]byte{index}, height) -} - -type op struct { - Code OperationCode - Pos navigation.Position -} - -type FakeBatchLoader struct { - cacheHeightLimit uint16 - cached map[string]*BatchNode - stored map[string]*BatchNode -} - -func NewFakeBatchLoader(cached map[string][]byte, stored map[string][]byte, cacheHeightLimit uint16) *FakeBatchLoader { - loader := &FakeBatchLoader{ - cacheHeightLimit: cacheHeightLimit, - cached: make(map[string]*BatchNode, 0), - stored: make(map[string]*BatchNode, 0), - } - for k, v := range cached { - loader.cached[k] = ParseBatchNode(1, v) - } - for k, v := range stored { - loader.stored[k] = ParseBatchNode(1, v) - } - return loader -} - -func (l *FakeBatchLoader) Load(pos navigation.Position) *BatchNode { - if pos.Height > l.cacheHeightLimit { - batch, ok := l.cached[pos.StringId()] - if !ok { - return NewEmptyBatchNode(len(pos.Index)) - } - return batch - } - batch, ok := l.stored[pos.StringId()] - if !ok { - return NewEmptyBatchNode(len(pos.Index)) - } - return batch -} diff --git a/balloon/hyper2/pruning2/verify.go b/balloon/hyper2/pruning2/verify.go deleted file mode 100644 index 048a3755a..000000000 --- a/balloon/hyper2/pruning2/verify.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pruning2 - -import ( - "bytes" - - "github.com/bbva/qed/balloon/hyper2/navigation" -) - -func PruneToVerify(index, value []byte, auditPathHeight uint16) *OperationsStack { - - var traverse func(pos navigation.Position, ops *OperationsStack) - - traverse = func(pos navigation.Position, ops *OperationsStack) { - - if pos.Height <= auditPathHeight { - ops.Push(leafHash(pos, value)) - return - } - - rightPos := pos.Right() - if bytes.Compare(index, rightPos.Index) < 0 { // go to left - traverse(pos.Left(), ops) - ops.Push(getFromPath(rightPos)) - } else { // go to right - ops.Push(getFromPath(pos.Left())) - traverse(rightPos, ops) - } - - ops.Push(innerHash(pos)) - - } - - ops := NewOperationsStack() - traverse(navigation.NewRootPosition(uint16(len(index)*8)), ops) - return ops - -} diff --git a/balloon/hyper2/tree.go b/balloon/hyper2/tree.go index 197ff4a4c..69c23b009 100644 --- a/balloon/hyper2/tree.go +++ b/balloon/hyper2/tree.go @@ -22,7 +22,7 @@ import ( "github.com/bbva/qed/balloon/hyper2/navigation" "github.com/bbva/qed/balloon/cache" - "github.com/bbva/qed/balloon/hyper2/pruning2" + "github.com/bbva/qed/balloon/hyper2/pruning" "github.com/bbva/qed/hashing" "github.com/bbva/qed/storage" "github.com/bbva/qed/util" @@ -36,7 +36,7 @@ type HyperTree struct { hasher hashing.Hasher cacheHeightLimit uint16 defaultHashes []hashing.Digest - batchLoader pruning2.BatchLoader + batchLoader pruning.BatchLoader sync.RWMutex } @@ -54,7 +54,7 @@ func NewHyperTree(hasherF func() hashing.Hasher, store storage.Store, cache cach hasher: hasher, cacheHeightLimit: cacheHeightLimit, defaultHashes: make([]hashing.Digest, numBits), - batchLoader: pruning2.NewDefaultBatchLoader(store, cache, cacheHeightLimit), + batchLoader: pruning.NewDefaultBatchLoader(store, cache, cacheHeightLimit), } tree.defaultHashes[0] = tree.hasher.Do([]byte{0x0}, []byte{0x0}) @@ -78,8 +78,8 @@ func (t *HyperTree) Add(eventDigest hashing.Digest, version uint64) (hashing.Dig versionAsBytes = versionAsBytes[len(versionAsBytes)-len(eventDigest):] // build a stack of operations and then interpret it to generate the root hash - ops := pruning2.PruneToInsert(eventDigest, versionAsBytes, t.cacheHeightLimit, t.batchLoader) - ctx := &pruning2.Context{ + ops := pruning.PruneToInsert(eventDigest, versionAsBytes, t.cacheHeightLimit, t.batchLoader) + ctx := &pruning.Context{ Hasher: t.hasher, Cache: t.cache, DefaultHashes: t.defaultHashes, @@ -104,8 +104,8 @@ func (t *HyperTree) QueryMembership(eventDigest hashing.Digest, version []byte) //log.Debugf("Proving membership for index %d with version %d", eventDigest, version) // build a stack of operations and then interpret it to generate the audit path - ops := pruning2.PruneToFind(eventDigest, t.batchLoader) - ctx := &pruning2.Context{ + ops := pruning.PruneToFind(eventDigest, t.batchLoader) + ctx := &pruning.Context{ Hasher: t.hasher, Cache: t.cache, DefaultHashes: t.defaultHashes,