-
Notifications
You must be signed in to change notification settings - Fork 0
/
kv.go
727 lines (644 loc) · 18.9 KB
/
kv.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
package alternator
import (
"bytes"
"encoding/gob"
"fmt"
"log"
"os"
"sync"
"time"
"github.com/boltdb/bolt"
)
// dataBucket stores all of this node's data.
var dataBucket = []byte("data")
// metaDataBucket stores all of this node's metadata.
var metaDataBucket = []byte("metadata")
// pendingDataBucket stores all pending PutData operations.
var pendingDataBucket = []byte("pendingdata")
// pendingMDBucket sotres all pending PutMetadata operations.
var pendingMDBucket = []byte("pendingmetadata")
// Metadata stores the metadata of a (key, value) pair.
type Metadata struct {
// Name of the entry, as used by the user. The key is the hash of the name.
Name string
// The list of replicators storing the value of the (key, value) pair.
Replicators []Key
}
/* Database struct and methods */
// DB is a wrapper around bolt's DB struct, so that methods can be added
type dB struct {
*bolt.DB
}
// getMDRange returns all key,value pairs in a given key range
func (db *dB) getMDRange(min, max Key) ([]Key, [][]byte) {
var keys []Key
var vals [][]byte
err := db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(metaDataBucket)
c := b.Cursor()
// Start seeking at min
c.Seek(min[:])
// Stop at max
for kslice, v := c.First(); kslice != nil && bytes.Compare(kslice, max[:]) < 0; kslice, v = c.Next() {
// TODO: Set a proper capacity for keys and vals?
keys = append(keys, SliceToKey(kslice))
vals = append(vals, v)
}
return nil
})
checkErr("Get range error error: ", err)
return keys, vals
}
// close closes the db
func (db *dB) close() {
db.Close()
}
// Initialize a node's database in the filesystem, create buckets
func (altNode *Node) initDB() {
checkBucketErr := func(err error) {
if (err != nil) && (err != bolt.ErrBucketExists) {
log.Fatal(err)
}
}
err := os.MkdirAll(altNode.Config.DotPath, 0777)
checkFatal("failed to make db path", err)
boltdb, err := bolt.Open(altNode.Config.DotPath+altNode.Port+".db", 0600, nil)
altNode.DB = &dB{boltdb}
checkFatal("failed to open .db file", err)
// Create a bucket for all entries
altNode.DB.Batch(func(tx *bolt.Tx) error {
// Create buckets
_, err = tx.CreateBucketIfNotExists(metaDataBucket)
checkBucketErr(err)
_, err := tx.CreateBucket(dataBucket)
checkBucketErr(err)
_, err = tx.CreateBucket(pendingMDBucket)
checkBucketErr(err)
_, err = tx.CreateBucket(pendingDataBucket)
checkBucketErr(err)
return nil
})
gob.Register(PutMDArgs{})
gob.Register(PutArgs{})
}
/* kv-store methods for PUTTING (inserting) into the database */
const (
pendingMD = iota
pendingData
)
// PendingPut stores information on some database operation that is still pending in the system.
type PendingPut struct {
Type int
Peers []Key
Args interface{}
}
// putPending puts the pending put in all of its pending peers
func (altNode *Node) putPending(k Key, pending PendingPut) {
successIndeces := make(map[int]struct{})
// TODO: make remote calls here async
for i, ownerID := range pending.Peers {
altNode.Members.RLock()
owner := getPeer(altNode.Members.Map[ownerID])
altNode.Members.RUnlock()
if owner == nil {
// Remove from Peers, node no longer in members list
successIndeces[i] = struct{}{}
continue
}
if pending.Type == pendingMD {
putMDArgs := pending.Args.(PutMDArgs)
err := altNode.rpcServ.MakeRemoteCall(owner, "PutMetadata", &putMDArgs, &struct{}{})
k := StringToKey(putMDArgs.Name)
if err == nil {
fmt.Printf("Good: md resolve, %v:%v\n", ownerID, k)
successIndeces[i] = struct{}{}
} else {
fmt.Printf("Bad: md resolve, %v:%v\n", ownerID, k)
}
} else if pending.Type == pendingData {
putArgs := pending.Args.(PutArgs)
err := altNode.rpcServ.MakeRemoteCall(owner, "PutData", &putArgs, &struct{}{})
k := StringToKey(putArgs.Name)
if err == nil {
fmt.Printf("Good: data resolve, %v:%v\n", ownerID, k)
successIndeces[i] = struct{}{}
} else {
fmt.Printf("Bad: data resolve %v:%v\n", ownerID, k)
}
}
}
// Remove successfully resolved entries from the pending Peers list
var newPending []Key
for i := range pending.Peers {
if _, ok := successIndeces[i]; !ok {
newPending = append(newPending, pending.Peers[i])
}
}
pending.Peers = newPending
altNode.DB.Batch(func(tx *bolt.Tx) error {
var b *bolt.Bucket
if pending.Type == pendingMD {
b = tx.Bucket(pendingMDBucket)
} else if pending.Type == pendingData {
b = tx.Bucket(pendingDataBucket)
} else {
return nil
}
if len(pending.Peers) == 0 {
b.Delete(k[:])
} else {
b.Put(k[:], serialize(pending))
}
return nil
})
}
func (altNode *Node) resolvePending() {
altNode.DB.Batch(func(tx *bolt.Tx) error {
pendMDBucket := tx.Bucket(pendingMDBucket)
c := pendMDBucket.Cursor()
for kslice, v := c.First(); kslice != nil; kslice, v = c.Next() {
k := SliceToKey(kslice)
PendingPut := bytesToPendingPut(v)
// Delete it while working on it
pendMDBucket.Delete(kslice)
go altNode.putPending(k, PendingPut)
}
pendDataBucket := tx.Bucket(pendingDataBucket)
c = pendDataBucket.Cursor()
for kslice, v := c.First(); kslice != nil; kslice, v = c.Next() {
k := SliceToKey(kslice)
PendingPut := bytesToPendingPut(v)
// Delete it while working on it
pendDataBucket.Delete(kslice)
go altNode.putPending(k, PendingPut)
}
return nil
})
return
}
func (altNode *Node) autoResolvePending() {
for {
time.Sleep(time.Duration(altNode.Config.ResolvePendingTime) * time.Millisecond)
altNode.resolvePending()
}
}
// PutArgs is a struct to represent the arguments of Put or DBPut
type PutArgs struct {
Name string
V []byte
Replicators []Key
Success int
}
// Put a (key, value) pair in the system
func (altNode *Node) Put(putArgs *PutArgs, _ *struct{}) error {
k := StringToKey(putArgs.Name)
var i int
// Attempt to pass the call to member of replication chain
chainLink := altNode.findListSuccessor(k)
for i = 0; i < altNode.Config.N; i++ {
// Wrap around if reach rings end
if chainLink == nil {
chainLink = altNode.Members.List.Front()
}
chainPeer := getPeer(chainLink)
// If this node is the chainLink
if chainPeer.ID == altNode.ID {
// Handle here
break
}
// Else try to pass the Put down the chain
fmt.Printf("Forwarding to %v\n", chainPeer.ID)
err := altNode.rpcServ.MakeRemoteCall(chainPeer, "Put", putArgs, &struct{}{})
if err == nil {
// Done, chain took responsibility
return nil
}
chainLink = chainLink.Next()
}
var wg sync.WaitGroup
putMDArgs := PutMDArgs{Name: putArgs.Name,
MD: Metadata{Name: putArgs.Name, Replicators: putArgs.Replicators}}
mdPeers := make([]*Peer, 0, altNode.Config.N)
mdPendingIDs := make([]Key, 0, altNode.Config.N)
dataPeers := make([]*Peer, 0, altNode.Config.N)
dataPendingIDs := make([]Key, 0, altNode.Config.N)
wg.Add(2)
go altNode.chainPutMetadata(&wg, &putMDArgs, &mdPeers, &mdPendingIDs)
go altNode.putDataInReps(&wg, putArgs, &dataPeers, &dataPendingIDs)
wg.Wait()
// Cancel puts if less than half the chain has metadata
// if len(mdPeers) < ((altNode.Config.N - 1) / 2) {
// altNode.undoPutMD(k, mdPeers)
// altNode.undoPutData(k, dataPeers)
// return ErrPutMDFail
// }
// Undo if put does not meet success criteria
// if ((putArgs.Success == 0) && (len(dataPeers) != len(putArgs.Replicators))) ||
// (len(dataPeers) < putArgs.Success) {
//
// fmt.Printf("Success is %v, len(dataPeers) is %v, len(reps) is %v, len(fails) is %v\n", putArgs.Success, len(dataPeers), len(putArgs.Replicators), len(dataPendingIDs))
//
// altNode.undoPutMD(k, mdPeers)
// altNode.undoPutData(k, dataPeers)
// return ErrPutFail
// }
go func() {
pendingMDPut := PendingPut{Type: pendingMD, Peers: mdPendingIDs, Args: putMDArgs}
pendingDataPut := PendingPut{Type: pendingData, Peers: dataPendingIDs, Args: putArgs}
altNode.DB.Batch(func(tx *bolt.Tx) error {
if len(pendingMDPut.Peers) > 0 {
pendMDBucket := tx.Bucket(pendingMDBucket)
pendMDBucket.Put(k[:], serialize(pendingMDPut))
}
if len(pendingDataPut.Peers) > 0 {
pendDataBucket := tx.Bucket(pendingDataBucket)
pendDataBucket.Put(k[:], serialize(pendingDataPut))
}
return nil
})
}()
return nil
}
func (altNode *Node) putDataInReps(wg *sync.WaitGroup, args *PutArgs, successPeers *[]*Peer,
failPeerIDs *[]Key) {
successCh := make(chan *Peer, len(args.Replicators))
failCh := make(chan Key, altNode.Config.N)
for _, repID := range args.Replicators {
altNode.Members.RLock()
currentLNode, ok := altNode.Members.Map[repID]
altNode.Members.RUnlock()
current := getPeer(currentLNode)
if !ok {
log.Print("Members map wrong error")
continue
}
go func(current *Peer) {
errChan := make(chan error, 1)
errChan <- altNode.rpcServ.MakeRemoteCall(current, "PutData", args, &struct{}{})
select {
case err := <-errChan:
if !checkErr("failed to put data in replicant", err) {
successCh <- current
} else {
altNode.rpcServ.CloseIfBad(err, current)
fmt.Println("Error returned!")
failCh <- current.ID
}
case <-time.After(time.Duration(altNode.Config.PutDataTimeout) * time.Millisecond):
fmt.Println("Timeout!")
failCh <- current.ID
}
}(current)
}
for i := 0; i < len(args.Replicators); i++ {
select {
case success := <-successCh:
*successPeers = append(*successPeers, success)
case fail := <-failCh:
*failPeerIDs = append(*failPeerIDs, fail)
}
}
wg.Done()
}
func (altNode *Node) chainPutMetadata(wg *sync.WaitGroup, putMDArgs *PutMDArgs,
successPeers *[]*Peer, failPeerIDs *[]Key) {
k := StringToKey(putMDArgs.Name)
successCh := make(chan *Peer, altNode.Config.N)
failCh := make(chan Key, altNode.Config.N)
i := 0
altNode.Members.RLock()
for current := altNode.Members.findSuccessor(k); i < altNode.Config.N; current = current.Next() {
if current == nil {
current = altNode.Members.List.Front()
}
currentPeer := getPeer(current)
go func(current *Peer) {
errChan := make(chan error, 1)
errChan <- altNode.rpcServ.MakeRemoteCall(currentPeer, "PutMetadata", &putMDArgs, &struct{}{})
select {
case err := <-errChan:
if !checkErr("metadata put fail", err) {
successCh <- currentPeer
} else {
failCh <- currentPeer.ID
altNode.rpcServ.CloseIfBad(err, current)
}
case <-time.After(time.Duration(altNode.Config.PutMDTimeout) * time.Millisecond):
failCh <- currentPeer.ID
}
}(currentPeer)
i++
}
altNode.Members.RUnlock()
for i := 0; i < altNode.Config.N; i++ {
select {
case success := <-successCh:
*successPeers = append(*successPeers, success)
case fail := <-failCh:
*failPeerIDs = append(*failPeerIDs, fail)
}
}
wg.Done()
return
}
// PutData puts the pair (hash(args.Name), args.V) in DB this node's database.
func (altNode *Node) PutData(args *PutArgs, _ *struct{}) error {
k := StringToKey(args.Name)
err := altNode.DB.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(dataBucket)
err := b.Put(k[:], args.V)
return err
})
if err == nil {
// log.Print("Stored pair " + args.Name + "," + string(args.V) + " key is " + k.String())
fmt.Println("Pair: " + k.String())
}
return err
}
// PutMDArgs represents the arguments of a call to PutMetadata
type PutMDArgs struct {
Name string
MD Metadata
}
// PutMetadata puts the (key, val) pair in DB
func (altNode *Node) PutMetadata(args *PutMDArgs, _ *struct{}) error {
k := StringToKey(args.Name)
// Serialize replicators
md := serialize(args.MD)
err := altNode.DB.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(metaDataBucket)
err := b.Put(k[:], md)
return err
})
if err == nil {
// log.Print("Stored metadata for " + args.Name + ", key is " + k.String())
fmt.Println("Meta: " + k.String())
}
return err
}
// RePutArgs are arguments for a call to RePut
type RePutArgs struct {
LeaverID Key
K Key
V []byte
}
// RePut reinserts an entry in this node into the system.
func (altNode *Node) RePut(args *RePutArgs, _ *struct{}) error {
successorLNode := altNode.getListSuccessor()
successor := getPeer(successorLNode)
if successor.ID == args.LeaverID {
successor = getPeer(successorLNode.Next())
if successor.ID == args.LeaverID {
log.Print("Cannot do RePut, successor leaving")
return ErrRePutFail
}
}
// Get metadata from chain
md, err := altNode.chainGetMetadata(args.K)
if checkErr("RePut failed", err) {
return err
}
// Find leaver
var i int
for i = range md.Replicators {
if md.Replicators[i] == args.LeaverID {
var randomID Key
// Get some random, different ID
for {
altNode.Members.RLock()
randomID = altNode.Members.getRandom().ID
altNode.Members.RUnlock()
if randomID != md.Replicators[i] {
break
}
}
// Replace
md.Replicators[i] = randomID
break
}
}
// Do the put again
putArgs := PutArgs{Name: md.Name, V: args.V, Replicators: md.Replicators, Success: 0}
err = altNode.rpcServ.MakeRemoteCall(successor, "Put", putArgs, &struct{}{})
checkErr("Put failed", err)
return err
}
// undoPutData undoes all data puts for a key in a set of nodes
func (altNode *Node) undoPutData(k Key, nodes []*Peer) error {
for _, node := range nodes {
altNode.rpcServ.MakeRemoteCall(node, "DropKeyData", k, &struct{}{})
}
// Wrong
return nil
}
// undoPutMD undoes all metadata puts for a key in a set of nodes
func (altNode *Node) undoPutMD(k Key, nodes []*Peer) error {
for _, node := range nodes {
altNode.rpcServ.MakeRemoteCall(node, "DropKeyMD", k, &struct{}{})
}
// Wrong
return nil
}
func (altNode *Node) rePutAllData() {
altNode.DB.View(func(tx *bolt.Tx) error {
b := tx.Bucket(dataBucket)
c := b.Cursor()
var wg sync.WaitGroup
for kslice, v := c.First(); kslice != nil; kslice, v = c.Next() {
var successor Peer
k := SliceToKey(kslice)
altNode.FindSuccessor(k, &successor)
rpArgs := RePutArgs{LeaverID: altNode.ID, K: k, V: v}
wg.Add(1)
go func(peer *Peer) {
err := altNode.rpcServ.MakeRemoteCall(&successor, "RePut", rpArgs, &struct{}{})
checkErr("Reput failed", err)
altNode.rpcServ.CloseIfBad(err, peer)
wg.Done()
}(&successor)
}
wg.Wait()
return nil
})
}
/* kv-store methods for GETTING from the database */
// GetMetadata returns the metadata (serialized) associated with a key in this node's database.
func (altNode *Node) GetMetadata(k Key, md *[]byte) error {
// fmt.Println("About to get metadata")
return altNode.DB.View(func(tx *bolt.Tx) error {
b := tx.Bucket(metaDataBucket)
*md = b.Get(k[:])
fmt.Println("Getting meta", k)
if md == nil {
return ErrKeyNotFound
}
return nil
})
}
// GetData returns the data associated with a key in this node's database.
func (altNode *Node) GetData(k Key, data *[]byte) error {
// fmt.Println("About to get data")
return altNode.DB.View(func(tx *bolt.Tx) error {
b := tx.Bucket(dataBucket)
*data = b.Get(k[:])
fmt.Println("Getting data", k)
if data == nil {
return ErrKeyNotFound
}
return nil
})
}
// Get sets 'value' to the value corresponding to the entry with name 'name'.
func (altNode *Node) Get(name string, value *[]byte) error {
k := StringToKey(name)
// Get replicators from metadata chain
md, err := altNode.chainGetMetadata(k)
if checkErr("Get failed", err) {
return err
}
// Get data from some replicant
for _, repID := range md.Replicators {
altNode.Members.RLock()
rep := getPeer(altNode.Members.Map[repID])
altNode.Members.RUnlock()
// In case rep left
// TODO: this occurs because when replicators leave, nobody re-replicates their data
if rep == nil {
continue
}
var result []byte
err := altNode.rpcServ.MakeRemoteCall(rep, "GetData", k, &result)
if err == nil {
*value = result
return nil
}
}
// None of the replicators had the data?
return ErrDataLost
}
// TODO: write chainGetData
func (altNode *Node) chainGetMetadata(k Key) (Metadata, error) {
// altNode.FindSuccessor(k, &successor)
i := 0
current := altNode.Members.findSuccessor(k)
for ; i < altNode.Config.N; current = current.Next() {
var rawMD []byte
if current == nil {
current = altNode.Members.List.Front()
}
currentPeer := getPeer(current)
call := altNode.rpcServ.MakeAsyncCall(currentPeer, "GetMetadata", k, &rawMD)
select {
case reply := <-call.Done:
if reply.Error == nil {
return bytesToMetadata(rawMD), nil
}
altNode.rpcServ.CloseIfBad(reply.Error, currentPeer)
case <-time.After(500 * time.Millisecond):
}
i++
}
return Metadata{}, ErrKeyNotFound
}
func (db *dB) batchPut(bucket []byte, keys *[]Key, vals *[][]byte) error {
anyError := false
err := db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(bucket)
for i := range *keys {
err := b.Put((*keys)[i][:], (*vals)[i])
if err != nil {
anyError = true
}
}
if anyError {
return ErrBatchPutIncomplete
}
return nil
})
return err
}
// BatchPutArgs are the arguments for a batch put
type BatchPutArgs struct {
// Put the (key, value) pairs in Bucket
Bucket []byte
Keys []Key
Vals [][]byte
}
// BatchPut puts a multiple of key-value pairs in the specified bucket.
func (altNode *Node) BatchPut(args BatchPutArgs, _ *struct{}) error {
err := altNode.DB.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(args.Bucket)
anyError := false
for i := range args.Keys {
// TODO: handle error
err := b.Put(args.Keys[i][:], args.Vals[i])
if err != nil {
anyError = true
}
log.Printf("Stored metadata for%s\n", args.Keys[i])
}
if anyError {
return ErrBatchPutIncomplete
}
return nil
})
return err
}
// DropKeyMD Drops all metadata associated with the key k.
func (altNode *Node) DropKeyMD(k *Key, _ *struct{}) error {
err := altNode.DB.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(metaDataBucket)
err := b.Delete(k[:])
return err
})
return err
}
// DropKeyData drops all data associated with the key k.
func (altNode *Node) DropKeyData(k *Key, _ *struct{}) error {
err := altNode.DB.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(dataBucket)
err := b.Delete(k[:])
return err
})
return err
}
// DumpMetadata dumps all keys in the metadata store to stdout.
func (altNode *Node) DumpMetadata(_ struct{}, _ *struct{}) error {
i := 0
fmt.Println("***Metadata dump***")
err := altNode.DB.View(func(tx *bolt.Tx) error {
b := tx.Bucket(metaDataBucket)
c := b.Cursor()
// Start seeking at min
c.Seek(MinKey[:])
// Stop at max
for kslice, _ := c.First(); kslice != nil; kslice, _ = c.Next() {
fmt.Printf("Meta: %v\n", SliceToKey(kslice))
i++
}
return nil
})
fmt.Printf("TOTAL: %d\n", i)
fmt.Println("***End of metadata dump***")
return err
}
// DumpData dumps all keys in the metadata store to stdout.
func (altNode *Node) DumpData(_ struct{}, _ *struct{}) error {
fmt.Println("***Data dump***")
i := 0
err := altNode.DB.View(func(tx *bolt.Tx) error {
b := tx.Bucket(dataBucket)
c := b.Cursor()
// Start seeking at min
c.Seek(MinKey[:])
// Stop at max
for kslice, _ := c.First(); kslice != nil; kslice, _ = c.Next() {
fmt.Printf("Data: %v\n", SliceToKey(kslice))
i++
}
return nil
})
fmt.Printf("TOTAL: %d\n", i)
fmt.Println("***End of data dump***")
return err
}