-
-
Notifications
You must be signed in to change notification settings - Fork 5.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
8bc4319
commit 833f8b9
Showing
195 changed files
with
221,830 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,5 +41,6 @@ coverage.out | |
| /dist | ||
| /custom | ||
| /data | ||
| /indexers | ||
| /log | ||
| /public/img/avatar | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| // Copyright 2017 The Gitea Authors. All rights reserved. | ||
| // Use of this source code is governed by a MIT-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| package models | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| "code.gitea.io/gitea/modules/log" | ||
| "code.gitea.io/gitea/modules/setting" | ||
| "code.gitea.io/gitea/modules/util" | ||
| "github.com/blevesearch/bleve" | ||
| "github.com/blevesearch/bleve/analysis/analyzer/simple" | ||
| "github.com/blevesearch/bleve/search/query" | ||
| ) | ||
|
|
||
| // issueIndexerUpdateQueue queue of issues that need to be updated in the issues | ||
| // indexer | ||
| var issueIndexerUpdateQueue chan *Issue | ||
|
|
||
| // issueIndexer (thread-safe) index for searching issues | ||
| var issueIndexer bleve.Index | ||
|
|
||
| // issueIndexerData data stored in the issue indexer | ||
| type issueIndexerData struct { | ||
| ID int64 | ||
| RepoID int64 | ||
|
|
||
| Title string | ||
| Content string | ||
| } | ||
|
|
||
| // numericQuery an numeric-equality query for the given value and field | ||
| func numericQuery(value int64, field string) *query.NumericRangeQuery { | ||
| f := float64(value) | ||
| tru := true | ||
| q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) | ||
| q.SetField(field) | ||
| return q | ||
| } | ||
|
|
||
| // SearchIssuesByKeyword searches for issues by given conditions. | ||
| // Returns the matching issue IDs | ||
| func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { | ||
| fields := strings.Fields(strings.ToLower(keyword)) | ||
| indexerQuery := bleve.NewConjunctionQuery( | ||
| numericQuery(repoID, "RepoID"), | ||
| bleve.NewDisjunctionQuery( | ||
| bleve.NewPhraseQuery(fields, "Title"), | ||
| bleve.NewPhraseQuery(fields, "Content"), | ||
| )) | ||
| search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false) | ||
| search.Fields = []string{"ID"} | ||
|
|
||
| result, err := issueIndexer.Search(search) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| issueIDs := make([]int64, len(result.Hits)) | ||
| for i, hit := range result.Hits { | ||
| issueIDs[i] = int64(hit.Fields["ID"].(float64)) | ||
| } | ||
| return issueIDs, nil | ||
| } | ||
|
|
||
| // InitIssueIndexer initialize issue indexer | ||
| func InitIssueIndexer() { | ||
| _, err := os.Stat(setting.Indexer.IssuePath) | ||
| if err != nil { | ||
| if os.IsNotExist(err) { | ||
| if err = createIssueIndexer(); err != nil { | ||
| log.Fatal(4, "CreateIssuesIndexer: %v", err) | ||
| } | ||
| if err = populateIssueIndexer(); err != nil { | ||
| log.Fatal(4, "PopulateIssuesIndex: %v", err) | ||
| } | ||
| } else { | ||
| log.Fatal(4, "InitIssuesIndexer: %v", err) | ||
| } | ||
| } else { | ||
| issueIndexer, err = bleve.Open(setting.Indexer.IssuePath) | ||
| if err != nil { | ||
| log.Fatal(4, "InitIssuesIndexer, open index: %v", err) | ||
| } | ||
| } | ||
| issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength) | ||
| go processIssueIndexerUpdateQueue() | ||
| // TODO close issueIndexer when Gitea closes | ||
| } | ||
|
|
||
| // createIssueIndexer create an issue indexer if one does not already exist | ||
| func createIssueIndexer() error { | ||
| mapping := bleve.NewIndexMapping() | ||
| docMapping := bleve.NewDocumentMapping() | ||
|
|
||
| docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping()) | ||
| docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping()) | ||
|
|
||
| textFieldMapping := bleve.NewTextFieldMapping() | ||
| textFieldMapping.Analyzer = simple.Name | ||
| docMapping.AddFieldMappingsAt("Title", textFieldMapping) | ||
| docMapping.AddFieldMappingsAt("Content", textFieldMapping) | ||
|
|
||
| mapping.AddDocumentMapping("issues", docMapping) | ||
|
|
||
| var err error | ||
| issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping) | ||
| return err | ||
| } | ||
|
|
||
| // populateIssueIndexer populate the issue indexer with issue data | ||
| func populateIssueIndexer() error { | ||
| for page := 1; ; page++ { | ||
| repos, err := Repositories(&SearchRepoOptions{ | ||
| Page: page, | ||
| PageSize: 10, | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("Repositories: %v", err) | ||
| } | ||
| if len(repos) == 0 { | ||
| return nil | ||
| } | ||
| batch := issueIndexer.NewBatch() | ||
| for _, repo := range repos { | ||
| issues, err := Issues(&IssuesOptions{ | ||
| RepoID: repo.ID, | ||
| IsClosed: util.OptionalBoolNone, | ||
| IsPull: util.OptionalBoolNone, | ||
| Page: -1, // do not page | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("Issues: %v", err) | ||
| } | ||
| for _, issue := range issues { | ||
| err = batch.Index(issue.indexUID(), issue.issueData()) | ||
| if err != nil { | ||
| return fmt.Errorf("batch.Index: %v", err) | ||
| } | ||
| } | ||
| } | ||
| if err = issueIndexer.Batch(batch); err != nil { | ||
| return fmt.Errorf("index.Batch: %v", err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func processIssueIndexerUpdateQueue() { | ||
| for { | ||
| select { | ||
| case issue := <-issueIndexerUpdateQueue: | ||
| if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil { | ||
| log.Error(4, "issuesIndexer.Index: %v", err) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // indexUID a unique identifier for an issue used in full-text indices | ||
| func (issue *Issue) indexUID() string { | ||
| return strconv.FormatInt(issue.ID, 36) | ||
| } | ||
|
|
||
| func (issue *Issue) issueData() *issueIndexerData { | ||
| return &issueIndexerData{ | ||
| ID: issue.ID, | ||
| RepoID: issue.RepoID, | ||
| Title: issue.Title, | ||
| Content: issue.Content, | ||
| } | ||
| } | ||
|
|
||
| // UpdateIssueIndexer add/update an issue to the issue indexer | ||
| func UpdateIssueIndexer(issue *Issue) { | ||
| go func() { | ||
| issueIndexerUpdateQueue <- issue | ||
| }() | ||
| } |
Oops, something went wrong.