Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes #117, HasA speedup #118

Merged
merged 3 commits into from
Aug 14, 2014
Merged

Fixes #117, HasA speedup #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
104e7d1
fix leveldb (and speed up hasa)
barakmich Aug 13, 2014
03798bc
fix mongo hasher
barakmich Aug 13, 2014
cfca7db
Go style
barakmich Aug 13, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions graph/iterator/hasa_iterator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,7 @@ func (it *HasA) Next() bool {
return graph.NextLogOut(it, 0, false)
}
tID := it.primaryIt.Result()
name := it.ts.Quad(tID).Get(it.dir)
val := it.ts.ValueOf(name)
val := it.ts.TripleDirection(tID, it.dir)
it.result = val
return graph.NextLogOut(it, val, true)
}
Expand Down
20 changes: 10 additions & 10 deletions graph/leveldb/iterator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ type Iterator struct {

func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
vb := value.(Token)
p := make([]byte, 0, 2+qs.hasher.Size())
p := make([]byte, 0, 2+qs.hasherSize)
p = append(p, []byte(prefix)...)
p = append(p, []byte(vb[1:])...)

Expand Down Expand Up @@ -169,31 +169,31 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
case quad.Subject:
return 2
case quad.Predicate:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Object:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Label:
return -1
}
}
if bytes.Equal(prefix, []byte("po")) {
switch d {
case quad.Subject:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Predicate:
return 2
case quad.Object:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Label:
return -1
}
}
if bytes.Equal(prefix, []byte("os")) {
switch d {
case quad.Subject:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Predicate:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Object:
return 2
case quad.Label:
Expand All @@ -203,11 +203,11 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("cp")) {
switch d {
case quad.Subject:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Predicate:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Object:
return 3*qs.hasher.Size() + 2
return 3*qs.hasherSize + 2
case quad.Label:
return 2
}
Expand Down
57 changes: 31 additions & 26 deletions graph/leveldb/triplestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,15 @@ func (t Token) Key() interface{} {
}

type TripleStore struct {
dbOpts *opt.Options
db *leveldb.DB
path string
open bool
size int64
hasher hash.Hash
writeopts *opt.WriteOptions
readopts *opt.ReadOptions
dbOpts *opt.Options
db *leveldb.DB
path string
open bool
size int64
hasherSize int
makeHasher func() hash.Hash
writeopts *opt.WriteOptions
readopts *opt.ReadOptions
}

func createNewLevelDB(path string, _ graph.Options) error {
Expand Down Expand Up @@ -93,7 +94,8 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro
write_buffer_mb = val
}
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.hasher = sha1.New()
qs.hasherSize = sha1.Size
qs.makeHasher = sha1.New
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
Expand Down Expand Up @@ -123,30 +125,33 @@ func (qs *TripleStore) Size() int64 {
}

func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*3))
hasher := qs.makeHasher()
key := make([]byte, 0, 2+(qs.hasherSize*3))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
return key
}

func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*4))
hasher := qs.makeHasher()
key := make([]byte, 0, 2+(qs.hasherSize*4))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{quad.Label.Prefix(), d[0].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(quad.Label))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(quad.Label), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
return key
}

func (qs *TripleStore) createValueKeyFor(s string) []byte {
key := make([]byte, 0, 1+qs.hasher.Size())
hasher := qs.makeHasher()
key := make([]byte, 0, 1+qs.hasherSize)
key = append(key, []byte("z")...)
key = append(key, qs.convertStringToByteHash(s)...)
key = append(key, qs.convertStringToByteHash(s, hasher)...)
return key
}

Expand Down Expand Up @@ -331,11 +336,11 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
return triple
}

func (qs *TripleStore) convertStringToByteHash(s string) []byte {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte {
hasher.Reset()
key := make([]byte, 0, qs.hasherSize)
hasher.Write([]byte(s))
key = hasher.Sum(key)
return key
}

Expand Down Expand Up @@ -441,7 +446,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
v := val.(Token)
offset := PositionOf(v[0:2], d, qs)
if offset != -1 {
return Token(append([]byte("z"), v[offset:offset+qs.hasher.Size()]...))
return Token(append([]byte("z"), v[offset:offset+qs.hasherSize]...))
} else {
return Token(qs.Quad(val).Get(d))
}
Expand Down
8 changes: 4 additions & 4 deletions graph/mongo/iterator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,13 +185,13 @@ func (it *Iterator) Contains(v graph.Value) bool {
case quad.Subject:
offset = 0
case quad.Predicate:
offset = (it.qs.hasher.Size() * 2)
offset = (it.qs.hasherSize * 2)
case quad.Object:
offset = (it.qs.hasher.Size() * 2) * 2
offset = (it.qs.hasherSize * 2) * 2
case quad.Label:
offset = (it.qs.hasher.Size() * 2) * 3
offset = (it.qs.hasherSize * 2) * 3
}
val := v.(string)[offset : it.qs.hasher.Size()*2+offset]
val := v.(string)[offset : it.qs.hasherSize*2+offset]
if val == it.hash {
it.result = v
return graph.ContainsLogOut(it, v, true)
Expand Down
42 changes: 23 additions & 19 deletions graph/mongo/triplestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,11 @@ var _ graph.BulkLoader = (*TripleStore)(nil)
const DefaultDBName = "cayley"

type TripleStore struct {
session *mgo.Session
db *mgo.Database
hasher hash.Hash
idCache *IDLru
session *mgo.Session
db *mgo.Database
hasherSize int
makeHasher func() hash.Hash
idCache *IDLru
}

func createNewMongoGraph(addr string, options graph.Options) error {
Expand Down Expand Up @@ -86,24 +87,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
}
qs.db = conn.DB(dbName)
qs.session = conn
qs.hasher = sha1.New()
qs.hasherSize = sha1.Size
qs.makeHasher = sha1.New
qs.idCache = NewIDLru(1 << 16)
return &qs, nil
}

func (qs *TripleStore) getIdForTriple(t quad.Quad) string {
id := qs.ConvertStringToByteHash(t.Subject)
id += qs.ConvertStringToByteHash(t.Predicate)
id += qs.ConvertStringToByteHash(t.Object)
id += qs.ConvertStringToByteHash(t.Label)
hasher := qs.makeHasher()
id := qs.convertStringToByteHash(t.Subject, hasher)
id += qs.convertStringToByteHash(t.Predicate, hasher)
id += qs.convertStringToByteHash(t.Object, hasher)
id += qs.convertStringToByteHash(t.Label, hasher)
return id
}

func (qs *TripleStore) ConvertStringToByteHash(s string) string {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) string {
hasher.Reset()
key := make([]byte, 0, qs.hasherSize)
hasher.Write([]byte(s))
key = hasher.Sum(key)
return hex.EncodeToString(key)
}

Expand Down Expand Up @@ -243,7 +246,8 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
}

func (qs *TripleStore) ValueOf(s string) graph.Value {
return qs.ConvertStringToByteHash(s)
h := qs.makeHasher()
return qs.convertStringToByteHash(s, h)
}

func (qs *TripleStore) NameOf(v graph.Value) string {
Expand Down Expand Up @@ -288,13 +292,13 @@ func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.V
case quad.Subject:
offset = 0
case quad.Predicate:
offset = (qs.hasher.Size() * 2)
offset = (qs.hasherSize * 2)
case quad.Object:
offset = (qs.hasher.Size() * 2) * 2
offset = (qs.hasherSize * 2) * 2
case quad.Label:
offset = (qs.hasher.Size() * 2) * 3
offset = (qs.hasherSize * 2) * 3
}
val := in.(string)[offset : qs.hasher.Size()*2+offset]
val := in.(string)[offset : qs.hasherSize*2+offset]
return val
}

Expand Down