Merge pull request #236 from barakmich/mongo_indexed_lto

Mongo Indexed LinksTo

graph/bolt/bolt_test.go

@@ -334,7 +334,7 @@ func TestSetIterator(t *testing.T) {
 	}
 	it.Reset()
 
-	and := iterator.NewAnd()
+	and := iterator.NewAnd(qs)
 	and.AddSubIterator(qs.QuadsAllIterator())
 	and.AddSubIterator(it)
 
@@ -354,7 +354,7 @@ func TestSetIterator(t *testing.T) {
 		t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
 	}
 
-	and = iterator.NewAnd()
+	and = iterator.NewAnd(qs)
 	and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B")))
 	and.AddSubIterator(it)
 
@@ -393,7 +393,7 @@ func TestSetIterator(t *testing.T) {
 	it.Reset()
 
 	// Order is important
-	and = iterator.NewAnd()
+	and = iterator.NewAnd(qs)
 	and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B")))
 	and.AddSubIterator(it)
 
@@ -406,7 +406,7 @@ func TestSetIterator(t *testing.T) {
 	it.Reset()
 
 	// Order is important
-	and = iterator.NewAnd()
+	and = iterator.NewAnd(qs)
 	and.AddSubIterator(it)
 	and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B")))
 

graph/gaedatastore/quadstore_test.go

@@ -282,12 +282,12 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
 
 	all := qs.NodesAllIterator()
 
-	innerAnd := iterator.NewAnd()
+	innerAnd := iterator.NewAnd(qs)
 	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed2, quad.Predicate))
 	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Object))
 
 	hasa := iterator.NewHasA(qs, innerAnd, quad.Subject)
-	outerAnd := iterator.NewAnd()
+	outerAnd := iterator.NewAnd(qs)
 	outerAnd.AddSubIterator(fixed)
 	outerAnd.AddSubIterator(hasa)
 

graph/iterator.go

@@ -30,6 +30,18 @@ type Tagger struct {
 	fixedTags map[string]Value
 }
 
+// TODO(barakmich): Linkage is general enough that there are places we take
+//the combined arguments `quad.Direction, graph.Value` that it may be worth
+//converting these into Linkages. If nothing else, future indexed iterators may
+//benefit from the shared representation
+
+// Linkage is a union type representing a set of values established for a given
+// quad direction.
+type Linkage struct {
+	Dir    quad.Direction
+	Values []Value
+}
+
 // Add a tag to the iterator.
 func (t *Tagger) Add(tag string) {
 	t.tags = append(t.tags, tag)

graph/iterator/and_iterator.go

@@ -31,13 +31,16 @@ type And struct {
 	result            graph.Value
 	runstats          graph.IteratorStats
 	err               error
+	qs                graph.QuadStore
 }
 
-// Creates a new And iterator.
-func NewAnd() *And {
+// NewAnd creates an And iterator. `qs` is only required when needing a handle
+// for QuadStore-specific optimizations, otherwise nil is acceptable.
+func NewAnd(qs graph.QuadStore) *And {
 	return &And{
 		uid:               NextUID(),
 		internalIterators: make([]graph.Iterator, 0, 20),
+		qs:                qs,
 	}
 }
 
@@ -79,7 +82,7 @@ func (it *And) TagResults(dst map[string]graph.Value) {
 }
 
 func (it *And) Clone() graph.Iterator {
-	and := NewAnd()
+	and := NewAnd(it.qs)
 	and.AddSubIterator(it.primaryIt.Clone())
 	and.tags.CopyFrom(it)
 	for _, sub := range it.internalIterators {

graph/iterator/and_iterator_optimize.go

@@ -78,7 +78,7 @@ func (it *And) Optimize() (graph.Iterator, bool) {
 
 	// The easiest thing to do at this point is merely to create a new And iterator
 	// and replace ourselves with our (reordered, optimized) clone.
-	newAnd := NewAnd()
+	newAnd := NewAnd(it.qs)
 
 	// Add the subiterators in order.
 	for _, sub := range its {
@@ -95,6 +95,18 @@ func (it *And) Optimize() (graph.Iterator, bool) {
 	// the new And (they were unchanged upon calling Optimize() on them, at the
 	// start).
 	it.cleanUp()
+
+	// Ask the graph.QuadStore if we can be replaced. Often times, this is a great
+	// optimization opportunity (there's a fixed iterator underneath us, for
+	// example).
+	if it.qs != nil {
+		newReplacement, hasOne := it.qs.OptimizeIterator(newAnd)
+		if hasOne {
+			newAnd.Close()
+			return newReplacement, true
+		}
+	}
+
 	return newAnd, true
 }
 

graph/iterator/and_iterator_optimize_test.go

@@ -26,10 +26,14 @@ import (
 )
 
 func TestIteratorPromotion(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	all := NewInt64(1, 3)
 	fixed := NewFixed(Identity)
 	fixed.Add(3)
-	a := NewAnd()
+	a := NewAnd(qs)
 	a.AddSubIterator(all)
 	a.AddSubIterator(fixed)
 	all.Tagger().Add("a")
@@ -51,9 +55,13 @@ func TestIteratorPromotion(t *testing.T) {
 }
 
 func TestNullIteratorAnd(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	all := NewInt64(1, 3)
 	null := NewNull()
-	a := NewAnd()
+	a := NewAnd(qs)
 	a.AddSubIterator(all)
 	a.AddSubIterator(null)
 	newIt, changed := a.Optimize()
@@ -66,11 +74,15 @@ func TestNullIteratorAnd(t *testing.T) {
 }
 
 func TestReorderWithTag(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	all := NewInt64(100, 300)
 	all.Tagger().Add("good")
 	all2 := NewInt64(1, 30000)
 	all2.Tagger().Add("slow")
-	a := NewAnd()
+	a := NewAnd(qs)
 	// Make all2 the default iterator
 	a.AddSubIterator(all2)
 	a.AddSubIterator(all)
@@ -92,11 +104,15 @@ func TestReorderWithTag(t *testing.T) {
 }
 
 func TestAndStatistics(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	all := NewInt64(100, 300)
 	all.Tagger().Add("good")
 	all2 := NewInt64(1, 30000)
 	all2.Tagger().Add("slow")
-	a := NewAnd()
+	a := NewAnd(qs)
 	// Make all2 the default iterator
 	a.AddSubIterator(all2)
 	a.AddSubIterator(all)

graph/iterator/and_iterator_test.go

@@ -23,10 +23,14 @@ import (
 
 // Make sure that tags work on the And.
 func TestTag(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	fix1 := NewFixed(Identity)
 	fix1.Add(234)
 	fix1.Tagger().Add("foo")
-	and := NewAnd()
+	and := NewAnd(qs)
 	and.AddSubIterator(fix1)
 	and.Tagger().Add("bar")
 	out := fix1.Tagger().Tags()
@@ -56,6 +60,10 @@ func TestTag(t *testing.T) {
 
 // Do a simple itersection of fixed values.
 func TestAndAndFixedIterators(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	fix1 := NewFixed(Identity)
 	fix1.Add(1)
 	fix1.Add(2)
@@ -65,7 +73,7 @@ func TestAndAndFixedIterators(t *testing.T) {
 	fix2.Add(3)
 	fix2.Add(4)
 	fix2.Add(5)
-	and := NewAnd()
+	and := NewAnd(qs)
 	and.AddSubIterator(fix1)
 	and.AddSubIterator(fix2)
 	// Should be as big as smallest subiterator
@@ -94,6 +102,10 @@ func TestAndAndFixedIterators(t *testing.T) {
 // If there's no intersection, the size should still report the same,
 // but there should be nothing to Next()
 func TestNonOverlappingFixedIterators(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	fix1 := NewFixed(Identity)
 	fix1.Add(1)
 	fix1.Add(2)
@@ -103,7 +115,7 @@ func TestNonOverlappingFixedIterators(t *testing.T) {
 	fix2.Add(5)
 	fix2.Add(6)
 	fix2.Add(7)
-	and := NewAnd()
+	and := NewAnd(qs)
 	and.AddSubIterator(fix1)
 	and.AddSubIterator(fix2)
 	// Should be as big as smallest subiterator
@@ -122,9 +134,13 @@ func TestNonOverlappingFixedIterators(t *testing.T) {
 }
 
 func TestAllIterators(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	all1 := NewInt64(1, 5)
 	all2 := NewInt64(4, 10)
-	and := NewAnd()
+	and := NewAnd(qs)
 	and.AddSubIterator(all2)
 	and.AddSubIterator(all1)
 
@@ -142,10 +158,14 @@ func TestAllIterators(t *testing.T) {
 }
 
 func TestAndIteratorErr(t *testing.T) {
+	qs := &store{
+		data: []string{},
+		iter: NewFixed(Identity),
+	}
 	wantErr := errors.New("unique")
 	allErr := newTestIterator(false, wantErr)
 
-	and := NewAnd()
+	and := NewAnd(qs)
 	and.AddSubIterator(allErr)
 	and.AddSubIterator(NewInt64(1, 5))
 

graph/iterator/query_shape_test.go

@@ -23,7 +23,7 @@ import (
 )
 
 func hasaWithTag(qs graph.QuadStore, tag string, target string) *HasA {
-	and := NewAnd()
+	and := NewAnd(qs)
 
 	obj := qs.FixedIterator()
 	obj.Add(qs.ValueOf(target))
@@ -91,7 +91,7 @@ func TestQueryShape(t *testing.T) {
 	}
 
 	// Given a name-of-an-and-iterator's shape.
-	andInternal := NewAnd()
+	andInternal := NewAnd(qs)
 
 	hasa1 := hasaWithTag(qs, "tag1", "cool")
 	hasa1.Tagger().Add("hasa1")
@@ -104,7 +104,7 @@ func TestQueryShape(t *testing.T) {
 	pred := qs.FixedIterator()
 	pred.Add(qs.ValueOf("name"))
 
-	and := NewAnd()
+	and := NewAnd(qs)
 	and.AddSubIterator(NewLinksTo(qs, andInternal, quad.Subject))
 	and.AddSubIterator(NewLinksTo(qs, pred, quad.Predicate))
 

graph/leveldb/leveldb_test.go

@@ -333,7 +333,7 @@ func TestSetIterator(t *testing.T) {
 	}
 	it.Reset()
 
-	and := iterator.NewAnd()
+	and := iterator.NewAnd(qs)
 	and.AddSubIterator(qs.QuadsAllIterator())
 	and.AddSubIterator(it)
 
@@ -353,7 +353,7 @@ func TestSetIterator(t *testing.T) {
 		t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
 	}
 
-	and = iterator.NewAnd()
+	and = iterator.NewAnd(qs)
 	and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B")))
 	and.AddSubIterator(it)
 
@@ -392,7 +392,7 @@ func TestSetIterator(t *testing.T) {
 	it.Reset()
 
 	// Order is important
-	and = iterator.NewAnd()
+	and = iterator.NewAnd(qs)
 	and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B")))
 	and.AddSubIterator(it)
 
@@ -405,7 +405,7 @@ func TestSetIterator(t *testing.T) {
 	it.Reset()
 
 	// Order is important
-	and = iterator.NewAnd()
+	and = iterator.NewAnd(qs)
 	and.AddSubIterator(it)
 	and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B")))
 

graph/memstore/quadstore_test.go

@@ -108,12 +108,12 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
 
 	all := qs.NodesAllIterator()
 
-	innerAnd := iterator.NewAnd()
+	innerAnd := iterator.NewAnd(qs)
 	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed2, quad.Predicate))
 	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, all, quad.Object))
 
 	hasa := iterator.NewHasA(qs, innerAnd, quad.Subject)
-	outerAnd := iterator.NewAnd()
+	outerAnd := iterator.NewAnd(qs)
 	outerAnd.AddSubIterator(fixed)
 	outerAnd.AddSubIterator(hasa)
 
@@ -193,7 +193,7 @@ func TestRemoveQuad(t *testing.T) {
 	fixed2 := qs.FixedIterator()
 	fixed2.Add(qs.ValueOf("follows"))
 
-	innerAnd := iterator.NewAnd()
+	innerAnd := iterator.NewAnd(qs)
 	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed, quad.Subject))
 	innerAnd.AddSubIterator(iterator.NewLinksTo(qs, fixed2, quad.Predicate))