-
Notifications
You must be signed in to change notification settings - Fork 385
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #116 from ponzu-cms/ponzu-dev
[core] Configurable full-text search with Bleve + search API endpoint
- Loading branch information
Showing
7 changed files
with
326 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package api | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "log" | ||
| "net/http" | ||
| "net/url" | ||
|
|
||
| "github.com/ponzu-cms/ponzu/system/db" | ||
| "github.com/ponzu-cms/ponzu/system/item" | ||
| ) | ||
|
|
||
| func searchContentHandler(res http.ResponseWriter, req *http.Request) { | ||
| qs := req.URL.Query() | ||
| t := qs.Get("type") | ||
| // type must be set, future version may compile multi-type result set | ||
| if t == "" { | ||
| res.WriteHeader(http.StatusBadRequest) | ||
| return | ||
| } | ||
|
|
||
| it, ok := item.Types[t] | ||
| if !ok { | ||
| res.WriteHeader(http.StatusBadRequest) | ||
| return | ||
| } | ||
|
|
||
| if hide(it(), res, req) { | ||
| return | ||
| } | ||
|
|
||
| q, err := url.QueryUnescape(qs.Get("q")) | ||
| if err != nil { | ||
| res.WriteHeader(http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // q must be set | ||
| if q == "" { | ||
| res.WriteHeader(http.StatusBadRequest) | ||
| return | ||
| } | ||
|
|
||
| // execute search for query provided, if no index for type send 404 | ||
| matches, err := db.SearchType(t, q) | ||
| if err == db.ErrNoSearchIndex { | ||
| res.WriteHeader(http.StatusNotFound) | ||
| return | ||
| } | ||
| if err != nil { | ||
| log.Println("[search] Error:", err) | ||
| res.WriteHeader(http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // respond with json formatted results | ||
| bb, err := db.ContentMulti(matches) | ||
| if err != nil { | ||
| log.Println("[search] Error:", err) | ||
| res.WriteHeader(http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // if we have matches, push the first as its matched by relevance | ||
| if len(bb) > 0 { | ||
| push(res, req, it, bb[0]) | ||
| } | ||
|
|
||
| var result = []json.RawMessage{} | ||
| for i := range bb { | ||
| result = append(result, bb[i]) | ||
| } | ||
|
|
||
| j, err := fmtJSON(result...) | ||
| if err != nil { | ||
| res.WriteHeader(http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| j, err = omit(it(), j) | ||
| if err != nil { | ||
| res.WriteHeader(http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| sendData(res, req, j) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/ponzu-cms/ponzu/system/item" | ||
|
|
||
| "encoding/json" | ||
|
|
||
| "github.com/blevesearch/bleve" | ||
| "github.com/blevesearch/bleve/mapping" | ||
| ) | ||
|
|
||
| var ( | ||
| // Search tracks all search indices to use throughout system | ||
| Search map[string]bleve.Index | ||
|
|
||
| // ErrNoSearchIndex is for failed checks for an index in Search map | ||
| ErrNoSearchIndex = errors.New("No search index found for type provided") | ||
| ) | ||
|
|
||
| // Searchable ... | ||
| type Searchable interface { | ||
| SearchMapping() (*mapping.IndexMappingImpl, error) | ||
| IndexContent() bool | ||
| } | ||
|
|
||
| func init() { | ||
| Search = make(map[string]bleve.Index) | ||
| } | ||
|
|
||
| // MapSearchIndex creates the mapping for a type and tracks the index to be used within | ||
| // the system for adding/deleting/checking data | ||
| func MapSearchIndex(typeName string) error { | ||
| // type assert for Searchable, get configuration (which can be overridden) | ||
| // by Ponzu user if defines own SearchMapping() | ||
| it, ok := item.Types[typeName] | ||
| if !ok { | ||
| return fmt.Errorf("[search] MapSearchIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName) | ||
| } | ||
| s, ok := it().(Searchable) | ||
| if !ok { | ||
| return fmt.Errorf("[search] MapSearchIndex Error: Item type %s doesn't implement db.Searchable", typeName) | ||
| } | ||
|
|
||
| // skip setting or using index for types that shouldn't be indexed | ||
| if !s.IndexContent() { | ||
| return nil | ||
| } | ||
|
|
||
| mapping, err := s.SearchMapping() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| idxName := typeName + ".index" | ||
| var idx bleve.Index | ||
|
|
||
| // check if index exists, use it or create new one | ||
| pwd, err := os.Getwd() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| searchPath := filepath.Join(pwd, "search") | ||
|
|
||
| err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| idxPath := filepath.Join(searchPath, idxName) | ||
| if _, err = os.Stat(idxPath); os.IsNotExist(err) { | ||
| idx, err = bleve.New(idxPath, mapping) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| idx.SetName(idxName) | ||
| } else { | ||
| idx, err = bleve.Open(idxPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| // add the type name to the index and track the index | ||
| Search[typeName] = idx | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // UpdateSearchIndex sets data into a content type's search index at the given | ||
| // identifier | ||
| func UpdateSearchIndex(id string, data interface{}) error { | ||
| // check if there is a search index to work with | ||
| target := strings.Split(id, ":") | ||
| ns := target[0] | ||
|
|
||
| idx, ok := Search[ns] | ||
| if ok { | ||
| // unmarshal json to struct, error if not registered | ||
| it, ok := item.Types[ns] | ||
| if !ok { | ||
| return fmt.Errorf("[search] UpdateSearchIndex Error: type '%s' doesn't exist", ns) | ||
| } | ||
|
|
||
| p := it() | ||
| err := json.Unmarshal(data.([]byte), &p) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // add data to search index | ||
| return idx.Index(id, p) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // DeleteSearchIndex removes data from a content type's search index at the | ||
| // given identifier | ||
| func DeleteSearchIndex(id string) error { | ||
| // check if there is a search index to work with | ||
| target := strings.Split(id, ":") | ||
| ns := target[0] | ||
|
|
||
| idx, ok := Search[ns] | ||
| if ok { | ||
| // add data to search index | ||
| return idx.Delete(id) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // SearchType conducts a search and returns a set of Ponzu "targets", Type:ID pairs, | ||
| // and an error. If there is no search index for the typeName (Type) provided, | ||
| // db.ErrNoSearchIndex will be returned as the error | ||
| func SearchType(typeName, query string) ([]string, error) { | ||
| idx, ok := Search[typeName] | ||
| if !ok { | ||
| return nil, ErrNoSearchIndex | ||
| } | ||
|
|
||
| q := bleve.NewQueryStringQuery(query) | ||
| req := bleve.NewSearchRequest(q) | ||
| res, err := idx.Search(req) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| var results []string | ||
| for _, hit := range res.Hits { | ||
| results = append(results, hit.ID) | ||
| } | ||
|
|
||
| return results, nil | ||
| } |
Oops, something went wrong.