From a50279f19adc3956be71937e34bcd8c4bdce1e4a Mon Sep 17 00:00:00 2001 From: zwwhdls Date: Tue, 17 Dec 2024 19:44:19 +0800 Subject: [PATCH 1/2] fix: split doc and attr into different index in meilisearch Signed-off-by: zwwhdls --- pkg/models/doc/document.go | 22 +++++++ pkg/store/meili/meili.go | 129 +++++++++++++++++++------------------ pkg/store/meili/model.go | 80 ++++++++++++++++------- 3 files changed, 146 insertions(+), 85 deletions(-) diff --git a/pkg/models/doc/document.go b/pkg/models/doc/document.go index eb02471..ccdb588 100644 --- a/pkg/models/doc/document.go +++ b/pkg/models/doc/document.go @@ -19,6 +19,9 @@ package doc import ( "fmt" "time" + + "github.com/basenana/friday/pkg/models" + "github.com/basenana/friday/pkg/utils" ) type Document struct { @@ -38,6 +41,25 @@ type Document struct { ChangedAt time.Time `json:"changed_at"` } +func (d *Document) NewTest() *Document { + return &Document{ + EntryId: 1, + Name: "test", + Namespace: models.DefaultNamespaceValue, + ParentEntryID: utils.ToPtr(int64(1)), + Source: "test", + Content: "test", + Summary: "test", + WebUrl: "test", + HeaderImage: "test", + SubContent: "test", + Marked: utils.ToPtr(true), + Unread: utils.ToPtr(true), + CreatedAt: time.Now(), + ChangedAt: time.Now(), + } +} + type DocumentFilter struct { Namespace string Search string diff --git a/pkg/store/meili/meili.go b/pkg/store/meili/meili.go index 4b05e46..5f421e5 100644 --- a/pkg/store/meili/meili.go +++ b/pkg/store/meili/meili.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" "github.com/meilisearch/meilisearch-go" @@ -66,82 +67,75 @@ func NewMeiliClient(conf config.Config) (store.DocStoreInterface, error) { } func (c *Client) init() error { - attrs, err := c.docIndex.GetFilterableAttributes() - if err != nil { + testDoc := (&doc.Document{}).NewTest() + // doc index + if err := c.initIndex(c.docIndex, DocFilterableAttrs, DocSortAttrs, func() error { + return c.CreateDocument(context.TODO(), testDoc) + }); err != nil { return err } - if !utils.Equal(DocFilterableAttrs, attrs) { - t, err := c.docIndex.UpdateFilterableAttributes(&DocFilterableAttrs) - if err != nil { - return err - } - if err = c.wait(context.TODO(), "document", t.TaskUID); err != nil { - return err - } - } - sortAttrs := DocSortAttrs - crtSortAttrs, err := c.docIndex.GetSortableAttributes() - if err != nil { + // attr index + if err := c.initIndex(c.attrIndex, DocAttrFilterableAttrs, DocAttrSortAttrs, func() error { + return c.CreateDocument(context.TODO(), testDoc) + }); err != nil { return err } - if !utils.Equal(sortAttrs, crtSortAttrs) { - t, err := c.docIndex.UpdateSortableAttributes(&sortAttrs) - if err != nil { + + d, err := c.GetDocument(context.TODO(), testDoc.EntryId) + if (err != nil && strings.Contains(err.Error(), "not found")) || (err == nil && d == nil) { + return nil + } else if d != nil { + return c.DeleteDocument(context.TODO(), testDoc.EntryId) + } + return err +} + +func (c *Client) initIndex(index meilisearch.IndexManager, filterableAttrs, sortableAttrs []string, noIndexFn func() error) error { + attrs, err := index.GetFilterableAttributes() + if err != nil { + if !strings.Contains(err.Error(), "not found") { return err } - if err = c.wait(context.TODO(), "document", t.TaskUID); err != nil { + if err := noIndexFn(); err != nil { return err } } - - // attr index - attrAttrs, err := c.attrIndex.GetFilterableAttributes() - if err != nil { - return err - } - if !utils.Equal(DocAttrFilterableAttrs, attrAttrs) { - t, err := c.docIndex.UpdateFilterableAttributes(&DocAttrFilterableAttrs) + if !utils.Equal(filterableAttrs, attrs) { + t, err := index.UpdateFilterableAttributes(&filterableAttrs) if err != nil { return err } - if err = c.wait(context.TODO(), "attr", t.TaskUID); err != nil { + if err = c.wait(context.TODO(), index, t.TaskUID); err != nil { return err } } - attrSortAttrs := DocAttrSortAttrs - crtAttrSortAttrs, err := c.docIndex.GetSortableAttributes() + + crtSortAttrs, err := index.GetSortableAttributes() if err != nil { return err } - if !utils.Equal(attrSortAttrs, crtAttrSortAttrs) { - t, err := c.docIndex.UpdateSortableAttributes(&attrSortAttrs) + if !utils.Equal(sortableAttrs, crtSortAttrs) { + t, err := index.UpdateSortableAttributes(&sortableAttrs) if err != nil { return err } - if err = c.wait(context.TODO(), "attr", t.TaskUID); err != nil { + if err = c.wait(context.TODO(), index, t.TaskUID); err != nil { return err } } return nil } -func (c *Client) index(kind string) meilisearch.IndexManager { - if kind == "attr" { - return c.attrIndex - } - return c.docIndex -} - func (c *Client) CreateDocument(ctx context.Context, doc *doc.Document) error { newDoc := (&Document{}).FromModel(doc) c.log.Debugf("store entryId %s", newDoc.EntryId) - task, err := c.index(newDoc.Kind).AddDocuments(newDoc, "id") + task, err := c.docIndex.AddDocuments(newDoc, "id") if err != nil { c.log.Error(err) return err } - if err := c.wait(ctx, newDoc.Kind, task.TaskUID); err != nil { + if err := c.wait(ctx, c.docIndex, task.TaskUID); err != nil { c.log.Errorf("store document with entryId %s error: %s", newDoc.EntryId, err) return err } @@ -149,12 +143,12 @@ func (c *Client) CreateDocument(ctx context.Context, doc *doc.Document) error { // store document attr newAttrs := (&DocumentAttrList{}).FromModel(doc) c.log.Debugf("store doc of entryId %d attrs: %s", doc.EntryId, newAttrs.String()) - t, err := c.index("attr").AddDocuments(newAttrs, "id") + t, err := c.attrIndex.AddDocuments(newAttrs, "id") if err != nil { c.log.Error(err) return err } - if err := c.wait(ctx, "attr", t.TaskUID); err != nil { + if err := c.wait(ctx, c.attrIndex, t.TaskUID); err != nil { c.log.Errorf("store document attr of entryId %d error: %s", doc.EntryId, err) return err } @@ -170,24 +164,24 @@ func (c *Client) UpdateDocument(ctx context.Context, doc *doc.Document) error { for _, aq := range newAttrsQuery.AttrQueries { filter = append(filter, aq.ToFilter()) } - t, err := c.index("attr").DeleteDocumentsByFilter(filter) + t, err := c.attrIndex.DeleteDocumentsByFilter(filter) if err != nil { c.log.Error(err) return err } - if err = c.wait(ctx, "attr", t.TaskUID); err != nil { + if err = c.wait(ctx, c.attrIndex, t.TaskUID); err != nil { c.log.Errorf("delete document by filter error: %s", err) return err } // store document attr newAttrs := (&DocumentAttrList{}).FromModel(doc) c.log.Debugf("store doc of entryId %d attrs: %s", doc.EntryId, newAttrs.String()) - t, err = c.index("attr").AddDocuments(newAttrs, "id") + t, err = c.attrIndex.AddDocuments(newAttrs, "id") if err != nil { c.log.Error(err) return err } - if err := c.wait(ctx, "attr", t.TaskUID); err != nil { + if err := c.wait(ctx, c.attrIndex, t.TaskUID); err != nil { c.log.Errorf("store document attr of entryId %d error: %s", doc.EntryId, err) return err } @@ -198,7 +192,7 @@ func (c *Client) GetDocument(ctx context.Context, entryId int64) (*doc.Document, namespace := models.GetNamespace(ctx) query := (&DocumentQuery{}).OfEntryId(namespace.String(), entryId) c.log.Debugf("get document by entryId: %d", entryId) - rep, err := c.index("document").Search("", query.ToRequest()) + rep, err := c.docIndex.Search("", query.ToRequest()) if err != nil { return nil, err } @@ -215,7 +209,7 @@ func (c *Client) GetDocument(ctx context.Context, entryId int64) (*doc.Document, // get attrs attrQuery := (&DocumentAttrQuery{}).OfEntryId(document.Namespace, document.EntryId) c.log.Debugf("filter document attr: %s", attrQuery.String()) - attrRep, err := c.index("attr").Search("", attrQuery.ToRequest()) + attrRep, err := c.attrIndex.Search("", attrQuery.ToRequest()) if err != nil { return nil, err } @@ -241,7 +235,7 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter attrQuery := (&DocumentAttrQueries{}).FromFilter(filter) for _, aq := range *attrQuery { c.log.Debugf("filter document attr: %s", aq.String()) - attrRep, err := c.index("attr").Search("", aq.ToRequest()) + attrRep, err := c.attrIndex.Search("", aq.ToRequest()) if err != nil { return nil, err } @@ -263,11 +257,14 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter Option: "IN", Value: entryIds, }) + } else { + // no result + return nil, nil } } c.log.Debugf("search document: [%s] query: [%s]", query.Search, query.String()) - rep, err := c.index("document").Search(query.Search, query.ToRequest()) + rep, err := c.docIndex.Search(query.Search, query.ToRequest()) if err != nil { return nil, err } @@ -282,16 +279,17 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter c.log.Errorf("unmarshal document error: %s", err) continue } + c.log.Debugf("get document: %s", document.String()) // get attrs attrQuery := (&DocumentAttrQuery{}).OfEntryId(document.Namespace, document.EntryId) c.log.Debugf("filter document attr: %s", attrQuery.String()) - attrRep, err := c.index("attr").Search("", attrQuery.ToRequest()) + attrRep, err := c.attrIndex.Search("", attrQuery.ToRequest()) if err != nil { return nil, err } - attrs := make([]*DocumentAttr, 0) + attrs := DocumentAttrList{} for _, hit := range attrRep.Hits { b, _ := json.Marshal(hit) attr := &DocumentAttr{} @@ -302,6 +300,7 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter } attrs = append(attrs, attr) } + c.log.Debugf("filter [%d] document attr: %s", len(attrs), attrs.String()) documents = append(documents, document.ToModel(attrs)) } return documents, nil @@ -309,23 +308,31 @@ func (c *Client) FilterDocuments(ctx context.Context, filter *doc.DocumentFilter func (c *Client) DeleteDocument(ctx context.Context, entryId int64) error { c.log.Debugf("delete document by entryId: %d", entryId) - aq := &AttrQuery{ - Attr: "entryId", - Option: "=", - Value: fmt.Sprintf("%d", entryId), + ns := models.GetNamespace(ctx) + dq := (&DocumentQuery{}).OfEntryId(ns.String(), entryId) + t, err := c.docIndex.DeleteDocumentsByFilter(dq.ToFilter()) + if err != nil { + c.log.Error(err) + return err } - t, err := c.index("attr").DeleteDocumentsByFilter(aq.ToFilter()) + if err := c.wait(ctx, c.docIndex, t.TaskUID); err != nil { + c.log.Errorf("delete document by filter error: %s", err) + } + + c.log.Debugf("delete document attr by entryId: %d", entryId) + aq := (&DocumentAttrQuery{}).OfEntryId(ns.String(), fmt.Sprintf("%d", entryId)) + t, err = c.attrIndex.DeleteDocumentsByFilter(aq.ToFilter()) if err != nil { c.log.Error(err) return err } - if err := c.wait(ctx, "attr", t.TaskUID); err != nil { + if err := c.wait(ctx, c.attrIndex, t.TaskUID); err != nil { c.log.Errorf("delete document by filter error: %s", err) } return nil } -func (c *Client) wait(ctx context.Context, kind string, taskUID int64) error { +func (c *Client) wait(ctx context.Context, index meilisearch.IndexManager, taskUID int64) error { t := time.NewTicker(100 * time.Millisecond) defer t.Stop() for { @@ -333,7 +340,7 @@ func (c *Client) wait(ctx context.Context, kind string, taskUID int64) error { case <-ctx.Done(): return fmt.Errorf("context timeout") case <-t.C: - t, err := c.index(kind).GetTask(taskUID) + t, err := index.GetTask(taskUID) if err != nil { c.log.Error(err) return err diff --git a/pkg/store/meili/model.go b/pkg/store/meili/model.go index 606120a..3ec3b66 100644 --- a/pkg/store/meili/model.go +++ b/pkg/store/meili/model.go @@ -26,6 +26,7 @@ import ( "github.com/meilisearch/meilisearch-go" "github.com/basenana/friday/pkg/models/doc" + "github.com/basenana/friday/pkg/utils" ) var ( @@ -119,9 +120,9 @@ func (d *Document) ToModel(attrs []*DocumentAttr) *doc.Document { pId := int64(parentID) m.ParentEntryID = &pId case "mark": - m.Marked = attr.Value.(*bool) + m.Marked = utils.ToPtr(attr.Value.(bool)) case "unRead": - m.Unread = attr.Value.(*bool) + m.Unread = utils.ToPtr(attr.Value.(bool)) } } return m @@ -166,6 +167,17 @@ func (d *DocumentAttr) String() string { return fmt.Sprintf("EntryId(%s) %s: %v", d.EntryId, d.Key, d.Value) } +func (d *DocumentAttr) NewTest() *DocumentAttr { + return &DocumentAttr{ + Id: "1", + Kind: "test", + Namespace: "test", + EntryId: "1", + Key: "test", + Value: "test", + } +} + type DocumentAttrList []*DocumentAttr func (d *DocumentAttrList) String() string { @@ -185,29 +197,33 @@ func (d *DocumentAttrList) FromModel(doc *doc.Document) *DocumentAttrList { Namespace: doc.Namespace, EntryId: fmt.Sprintf("%d", doc.EntryId), Key: "parentId", - Value: doc.ParentEntryID, + Value: fmt.Sprintf("%d", *doc.ParentEntryID), }) } + mark := false if doc.Marked != nil { - attrs = append(attrs, &DocumentAttr{ - Id: uuid.New().String(), - Kind: "attr", - Namespace: doc.Namespace, - EntryId: fmt.Sprintf("%d", doc.EntryId), - Key: "mark", - Value: doc.Marked, - }) - } + mark = *doc.Marked + } + attrs = append(attrs, &DocumentAttr{ + Id: uuid.New().String(), + Kind: "attr", + Namespace: doc.Namespace, + EntryId: fmt.Sprintf("%d", doc.EntryId), + Key: "mark", + Value: mark, + }) + unread := true if doc.Unread != nil { - attrs = append(attrs, &DocumentAttr{ - Id: uuid.New().String(), - Kind: "attr", - Namespace: doc.Namespace, - EntryId: fmt.Sprintf("%d", doc.EntryId), - Key: "unRead", - Value: doc.Unread, - }) - } + unread = *doc.Unread + } + attrs = append(attrs, &DocumentAttr{ + Id: uuid.New().String(), + Kind: "attr", + Namespace: doc.Namespace, + EntryId: fmt.Sprintf("%d", doc.EntryId), + Key: "unRead", + Value: unread, + }) return (*DocumentAttrList)(&attrs) } @@ -413,6 +429,14 @@ func (q *DocumentAttrQuery) ToRequest() *meilisearch.SearchRequest { } } +func (q *DocumentAttrQuery) ToFilter() interface{} { + filter := []interface{}{} + for _, aq := range q.AttrQueries { + filter = append(filter, aq.ToFilter()) + } + return filter +} + type DocumentAttrQueries []*DocumentAttrQuery func (q *DocumentAttrQueries) FromFilter(query *doc.DocumentFilter) *DocumentAttrQueries { @@ -438,7 +462,7 @@ func (q *DocumentAttrQueries) FromFilter(query *doc.DocumentFilter) *DocumentAtt { Attr: "value", Option: "=", - Value: query.ParentID, + Value: *query.ParentID, }, }, }) @@ -464,7 +488,7 @@ func (q *DocumentAttrQueries) FromFilter(query *doc.DocumentFilter) *DocumentAtt { Attr: "value", Option: "=", - Value: query.Marked, + Value: *query.Marked, }, }, }) @@ -490,7 +514,7 @@ func (q *DocumentAttrQueries) FromFilter(query *doc.DocumentFilter) *DocumentAtt { Attr: "value", Option: "=", - Value: query.Unread, + Value: *query.Unread, }, }, }) @@ -529,6 +553,14 @@ func (q *DocumentQuery) String() string { return fmt.Sprintf("search: [%s], attr query: [%s]", q.Search, filters) } +func (q *DocumentQuery) ToFilter() interface{} { + filter := []interface{}{} + for _, aq := range q.AttrQueries { + filter = append(filter, aq.ToFilter()) + } + return filter +} + func (q *DocumentQuery) ToRequest() *meilisearch.SearchRequest { // build filter filter := []interface{}{} From e397d1267abe33672de6caf66a6eeb5981913d05 Mon Sep 17 00:00:00 2001 From: zwwhdls Date: Tue, 17 Dec 2024 20:37:02 +0800 Subject: [PATCH 2/2] fix unittest Signed-off-by: zwwhdls --- pkg/store/meili/mock.go | 38 +++++++++++++------------------------- pkg/store/meili/model.go | 2 +- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pkg/store/meili/mock.go b/pkg/store/meili/mock.go index dd5dce4..b0f3b21 100644 --- a/pkg/store/meili/mock.go +++ b/pkg/store/meili/mock.go @@ -113,39 +113,31 @@ func (m *MockClient) FilterDocuments(ctx context.Context, filter *doc.DocumentFi all := len(aq.AttrQueries) matched := true for _, q := range aq.AttrQueries { - if q.Attr == "entryId" { + switch q.Attr { + case "entryId": all -= 1 if !match(q, attr.EntryId) { matched = false - continue } - } - if q.Attr == "namespace" { + case "namespace": all -= 1 if !match(q, attr.Namespace) { matched = false - continue } - } - if q.Attr == "key" { + case "key": all -= 1 if !match(q, attr.Key) { matched = false - continue } - } - if q.Attr == "value" { + case "value": all -= 1 if !match(q, attr.Value) { matched = false - continue } - } - if q.Attr == "kind" { + case "kind": all -= 1 if !match(q, attr.Kind) { matched = false - continue } } } @@ -160,6 +152,8 @@ func (m *MockClient) FilterDocuments(ctx context.Context, filter *doc.DocumentFi Option: "IN", Value: entryId, }) + } else { + return nil, nil } } @@ -168,32 +162,26 @@ func (m *MockClient) FilterDocuments(ctx context.Context, filter *doc.DocumentFi matched := true all := len(query.AttrQueries) for _, q := range query.AttrQueries { - if q.Attr == "entryId" { + switch q.Attr { + case "entryId": all -= 1 if !match(q, d.EntryId) { matched = false - continue } - } - if q.Attr == "namespace" { + case "namespace": all -= 1 if !match(q, d.Namespace) { matched = false - continue } - } - if q.Attr == "id" { + case "id": all -= 1 if !match(q, d.Id) { matched = false - continue } - } - if q.Attr == "kind" { + case "kind": all -= 1 if !match(q, d.Kind) { matched = false - continue } } } diff --git a/pkg/store/meili/model.go b/pkg/store/meili/model.go index 3ec3b66..5d15bf3 100644 --- a/pkg/store/meili/model.go +++ b/pkg/store/meili/model.go @@ -197,7 +197,7 @@ func (d *DocumentAttrList) FromModel(doc *doc.Document) *DocumentAttrList { Namespace: doc.Namespace, EntryId: fmt.Sprintf("%d", doc.EntryId), Key: "parentId", - Value: fmt.Sprintf("%d", *doc.ParentEntryID), + Value: *doc.ParentEntryID, }) } mark := false