Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

api cleanup: #181

Merged
merged 4 commits into from
Jan 4, 2019
Merged

api cleanup: #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
01f6fd9
api cleanup: remove dedicated aggregation endpoint, dont stream indic…
adamstruck Dec 31, 2018
d9aece8
cleanup labels conformance test
adamstruck Jan 3, 2019
6ec4ccf
fix auth test
adamstruck Jan 3, 2019
ff1c065
minor change to list labels endpoint
adamstruck Jan 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions cmd/kvload/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ package kvload

import (
"fmt"
"log"

"github.com/bmeg/golib"
"github.com/bmeg/grip/gripql"
"github.com/bmeg/grip/kvgraph"
"github.com/bmeg/grip/kvi"
"github.com/bmeg/grip/util"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

Expand Down Expand Up @@ -88,7 +88,7 @@ var Cmd = &cobra.Command{
}

for _, vertexFile := range vertexFileArray {
log.Printf("Loading %s", vertexFile)
log.Infof("Loading %s", vertexFile)
count := 0
vertexChan := make(chan []*gripql.Vertex, 100)
vertexBatch := make([]*gripql.Vertex, 0, batchSize)
Expand All @@ -101,20 +101,20 @@ var Cmd = &cobra.Command{
}
count++
if count%10000 == 0 {
log.Printf("Loaded %d vertices", count)
log.Infof("Loaded %d vertices", count)
}
}
if len(vertexBatch) > 0 {
vertexChan <- vertexBatch
}
log.Printf("Loaded %d vertices", count)
log.Infof("Loaded %d vertices", count)
close(vertexChan)
}()

for batch := range vertexChan {
//serialize and store vertex
if err := kgraph.AddVertex(batch); err != nil {
log.Printf("%s", err)
log.Errorf("%s", err)
}
}
}
Expand All @@ -133,19 +133,19 @@ var Cmd = &cobra.Command{
}
count++
if count%10000 == 0 {
log.Printf("Loaded %d edges", count)
log.Infof("Loaded %d edges", count)
}
}
if len(edgeBatch) > 0 {
edgeChan <- edgeBatch
}
log.Printf("Loaded %d edges", count)
log.Infof("Loaded %d edges", count)
close(edgeChan)
}()
for batch := range edgeChan {
//serialize and store vertex
if err := kgraph.AddEdge(batch); err != nil {
log.Printf("%s", err)
log.Errorf("%s", err)
}
}
}
Expand Down
6 changes: 4 additions & 2 deletions cmd/list/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@ var Cmd = &cobra.Command{
if err != nil {
return err
}
graphs, err := conn.ListGraphs()

resp, err := conn.ListGraphs()
if err != nil {
return err
}
for g := range graphs {

for _, g := range resp.Graphs {
fmt.Printf("%s\n", g)
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions cmd/load/example/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@ var Cmd = &cobra.Command{
return err
}

graphCh, err := conn.ListGraphs()
resp, err := conn.ListGraphs()
if err != nil {
return err
}
graphs := []string{}
for g := range graphCh {
for _, g := range resp.Graphs {
graphs = append(graphs, g)
}
if found(graphs, graph) {
Expand Down
4 changes: 2 additions & 2 deletions cmd/load/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,13 @@ var Cmd = &cobra.Command{
return err
}

graphs, err := conn.ListGraphs()
resp, err := conn.ListGraphs()
if err != nil {
return err
}

found := false
for g := range graphs {
for _, g := range resp.Graphs {
if graph == g {
found = true
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/mongoload/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,12 @@ var Cmd = &cobra.Command{
if err != nil {
return err
}
graphs, err := conn.ListGraphs()
resp, err := conn.ListGraphs()
if err != nil {
return err
}
found := false
for g := range graphs {
for _, g := range resp.Graphs {
if graph == g {
found = true
}
Expand Down
162 changes: 3 additions & 159 deletions conformance/tests/ot_aggregations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,65 +32,12 @@ def setupGraph(O):
O.addEdge("4", "5", "created", {"weight": 1.0})


def test_term_aggregation(O):
errors = []
setupGraph(O)

response = O.aggregate(gripql.term("test-agg", "Person", "name", 2))
if 'test-agg' not in response:
errors.append("Result had Incorrect aggregation name")
return errors
response = response['test-agg']

if len(response["buckets"]) != 2:
errors.append(
"Unexpected number of terms: %d != %d" %
(len(response["buckets"]), 2)
)

for res in response["buckets"]:
if res["key"] not in ["marko", "alex"]:
errors.append(
"Incorrect term returned: %s" % (res["key"])
)
if res["value"] != 2:
errors.append(
"Incorrect term count: %d != %d" %
(res["value"], 2))

response = O.aggregate(gripql.term("test-agg-no-limit", "Person", "name", size=None))
if 'test-agg-no-limit' not in response:
errors.append("Result had Incorrect aggregation name")
return errors
response = response['test-agg-no-limit']

if len(response["buckets"]) != 8:
errors.append(
"Unexpected number of terms: %d != %d" %
(len(response["buckets"]), 8)
)

for res in response["buckets"]:
if res["key"] in ["marko", "alex"]:
if res["value"] != 2:
errors.append(
"Incorrect term count: %d != %d" %
(res["value"], 2))
else:
if res["value"] != 1:
errors.append(
"Incorrect term count: %d != %d" %
(res["value"], 1))

return errors


def test_traversal_term_aggregation(O):
errors = []
setupGraph(O)

count = 0
for row in O.query().V("1").out().aggregate(gripql.term("traversal-agg", "Person", "name")):
for row in O.query().V("1").out().hasLabel("Person").aggregate(gripql.term("traversal-agg", "name")):
if 'traversal-agg' not in row:
errors.append("Result had Incorrect aggregation name")
return errors
Expand Down Expand Up @@ -123,53 +70,12 @@ def test_traversal_term_aggregation(O):
return errors


def test_histogram_aggregation(O):
errors = []
setupGraph(O)

response = O.aggregate(gripql.histogram("test-agg", "Person", "age", 5))
if 'test-agg' not in response:
errors.append("Result had Incorrect aggregation name")
return errors
response = response['test-agg']

if len(response["buckets"]) != 6:
errors.append(
"Unexpected number of terms: %d != %d" %
(len(response["buckets"]), 6)
)

for res in response["buckets"]:
if res["key"] == 20:
if res["value"] != 1:
errors.append("Incorrect bucket count returned: %s" % res)
elif res["key"] == 25:
if res["value"] != 3:
errors.append("Incorrect bucket count returned: %s" % res)
elif res["key"] == 30:
if res["value"] != 2:
errors.append("Incorrect bucket count returned: %s" % res)
elif res["key"] == 35:
if res["value"] != 2:
errors.append("Incorrect bucket count returned: %s" % res)
elif res["key"] == 40:
if res["value"] != 1:
errors.append("Incorrect bucket count returned: %s" % res)
elif res["key"] == 45:
if res["value"] != 1:
errors.append("Incorrect bucket count returned: %s" % res)
else:
errors.append("Incorrect bucket key returned: %s" % res)

return errors


def test_traversal_histogram_aggregation(O):
errors = []
setupGraph(O)

count = 0
for row in O.query().V("1").out().aggregate(gripql.histogram("traversal-agg", "Person", "age", 5)):
for row in O.query().V("1").out().hasLabel("Person").aggregate(gripql.histogram("traversal-agg", "age", 5)):
count += 1
if 'traversal-agg' not in row:
errors.append("Result had Incorrect aggregation name")
Expand Down Expand Up @@ -209,75 +115,13 @@ def test_traversal_histogram_aggregation(O):
return errors


def test_percentile_aggregation(O):
errors = []
setupGraph(O)

percents = [1, 5, 25, 50, 75, 95, 99, 99.9]
response = O.aggregate(gripql.percentile("test-agg", "Person", "age", percents))
if 'test-agg' not in response:
errors.append("Result had Incorrect aggregation name")
return errors
response = response['test-agg']

if len(response["buckets"]) != len(percents):
errors.append(
"Unexpected number of terms: %d != %d" %
(len(response["buckets"]), len(percents))
)

ages = np.array([29, 41, 25, 32, 35, 30, 45, 26, 22, 36])

# for tests quantiles need to be withing 10% of the actual value
def getMinMax(input_data, percent, accuracy=0.1):
return np.percentile(input_data, percent) * (1 - accuracy), np.percentile(input_data, percent) * (1 + accuracy)

for res in response["buckets"]:
if res["key"] == 1:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 5:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 25:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 50:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 75:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 95:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 99:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
elif res["key"] == 99.9:
minpv, maxpv = getMinMax(ages, res["key"])
if res["value"] <= minpv or res["value"] >= maxpv:
errors.append("Incorrect quantile value returned for %.2f:\n\tmin: %.2f\n\tmax: %.2f\n\tactual: %.2f" % (res["key"], minpv, maxpv, res["value"]))
else:
errors.append("Incorrect bucket key returned: %s" % res)

return errors


def test_traversal_percentile_aggregation(O):
errors = []
setupGraph(O)

count = 0
percents = [1, 5, 25, 50, 75, 95, 99, 99.9]
for row in O.query().V("1").out().aggregate(gripql.percentile("traversal-agg", "Person", "age", percents)):
for row in O.query().V("1").out().hasLabel("Person").aggregate(gripql.percentile("traversal-agg", "age", percents)):
count += 1

if 'traversal-agg' not in row:
Expand Down
6 changes: 3 additions & 3 deletions conformance/tests/ot_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ def test_index(O):
O.addEdge("6", "3", "created", {"weight": 0.2})
O.addEdge("4", "5", "created", {"weight": 1.0})

resp = O.listIndices()
found = False
for i in O.listIndices():
if i['field'] == 'name' and i['label'] == 'Person':
for i in resp:
if i["field"] == "name" and i["label"] == "Person":
found = True

if not found:
errors.append("Expected index not found")

Expand Down
32 changes: 32 additions & 0 deletions conformance/tests/ot_labels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@


def test_list_labels(O):
errors = []

O.addVertex("1", "Person", {"name": "marko", "age": "29"})
O.addVertex("2", "Person", {"name": "vadas", "age": "27"})
O.addVertex("3", "Software", {"name": "lop", "lang": "java"})
O.addVertex("4", "Person", {"name": "josh", "age": "32"})
O.addVertex("5", "Software", {"name": "ripple", "lang": "java"})
O.addVertex("6", "Person", {"name": "peter", "age": "35"})
O.addVertex("7", "Person", {"name": "marko", "age": "35"})

O.addEdge("1", "3", "created", {"weight": 0.4})
O.addEdge("1", "2", "knows", {"weight": 0.5})
O.addEdge("1", "4", "knows", {"weight": 1.0})
O.addEdge("4", "3", "created", {"weight": 0.4})
O.addEdge("6", "3", "created", {"weight": 0.2})
O.addEdge("4", "5", "created", {"weight": 1.0})

resp = O.listLabels()
print(resp)
if len(resp["vertex_labels"]) != 2:
errors.append("listLabels returned an unexpected number of vertex labels; %d != 2" % (len(resp["vertex_labels"])))
if sorted(resp["vertex_labels"]) != ["Person", "Software"]:
errors.append("listLabels returned unexpected vertex labels")
if len(resp["edge_labels"]) != 2:
errors.append("listLabels returned an unexpected number of edge labels; %d != 2" % (len(resp["edge_labels"])))
if sorted(resp["edge_labels"]) != ["created", "knows"]:
errors.append("listLabels returned unexpected edge labels")

return errors
Loading