From d508e053f2f27031c4b28d4e5f66460e902f2024 Mon Sep 17 00:00:00 2001 From: zwwhdls Date: Sat, 16 Dec 2023 14:39:17 +0800 Subject: [PATCH] skip summary if content has less than 300 token Signed-off-by: zwwhdls --- go.mod | 2 +- go.sum | 2 ++ pkg/plugin/buildin/summary.go | 9 ++++++++- .../friday/pkg/vectorstore/postgres/migrate.go | 11 ----------- vendor/modules.txt | 2 +- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index bcf46c02..edf5b198 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.26 github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 github.com/aws/smithy-go v1.13.5 - github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748 + github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8 github.com/blevesearch/bleve/v2 v2.3.10 github.com/bluele/gcache v0.0.2 github.com/getsentry/sentry-go v0.22.0 diff --git a/go.sum b/go.sum index 20477c9b..31c89a24 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/basenana/friday v0.0.0-20231207142041-8f7eb86afc78 h1:PPj10ZL8OUIzT2w github.com/basenana/friday v0.0.0-20231207142041-8f7eb86afc78/go.mod h1:OGCCDNAXqN/ZTqO8EsaGGUBt5GrVwb/BVdnDQA4KbZI= github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748 h1:STmXI0Q6G3P55zZJRpkgJKQrcyb3/bTgKD/f2QJSU2Y= github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748/go.mod h1:OGCCDNAXqN/ZTqO8EsaGGUBt5GrVwb/BVdnDQA4KbZI= +github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8 h1:jGuzKXoNJ4lIHe/SJmCfMZWSmo3DdDu7evZWNmxXkcs= +github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8/go.mod h1:OGCCDNAXqN/ZTqO8EsaGGUBt5GrVwb/BVdnDQA4KbZI= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= diff --git a/pkg/plugin/buildin/summary.go b/pkg/plugin/buildin/summary.go index ae44ad37..fbed12d5 100644 --- a/pkg/plugin/buildin/summary.go +++ b/pkg/plugin/buildin/summary.go @@ -21,6 +21,7 @@ import ( "context" "fmt" + "github.com/basenana/friday/pkg/utils/files" "go.uber.org/zap" "github.com/basenana/nanafs/pkg/friday" @@ -33,6 +34,7 @@ import ( const ( SummaryPluginName = "summary" SummaryPluginVersion = "1.0" + DefaultSummaryToken = 300 ) type SummaryPlugin struct { @@ -84,7 +86,12 @@ func (i *SummaryPlugin) Run(ctx context.Context, request *pluginapi.Request) (*p } trimmedContent := utils.ContentTrim(docType, buf.String()) - i.logger(ctx).Infow("get docs", "length", len(trimmedContent), "entryId", request.EntryId) + length := files.Length(trimmedContent) + if length <= DefaultSummaryToken { + i.logger(ctx).Infow("skip summary, length less than default summary token.", "length", length, "default token", DefaultSummaryToken, "entryId", request.EntryId) + return pluginapi.NewResponseWithResult(map[string]any{}), nil + } + i.logger(ctx).Infow("get docs", "length", length, "entryId", request.EntryId) summary, usage, err := friday.SummaryFile(ctx, fmt.Sprintf("entry_%d", request.EntryId), trimmedContent) if err != nil { return pluginapi.NewFailedResponse(fmt.Sprintf("summary documents failed: %s", err)), nil diff --git a/vendor/github.com/basenana/friday/pkg/vectorstore/postgres/migrate.go b/vendor/github.com/basenana/friday/pkg/vectorstore/postgres/migrate.go index e58e9283..cd301234 100644 --- a/vendor/github.com/basenana/friday/pkg/vectorstore/postgres/migrate.go +++ b/vendor/github.com/basenana/friday/pkg/vectorstore/postgres/migrate.go @@ -34,17 +34,6 @@ func buildMigrations() []*gormigrate.Migration { return nil }, }, - { - ID: "2023111900", - Migrate: func(db *gorm.DB) error { - return db.AutoMigrate( - &BleveKV{}, - ) - }, - Rollback: func(db *gorm.DB) error { - return nil - }, - }, } } diff --git a/vendor/modules.txt b/vendor/modules.txt index d6264202..ba5cd536 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -128,7 +128,7 @@ github.com/aws/smithy-go/waiter ## explicit github.com/aymerick/douceur/css github.com/aymerick/douceur/parser -# github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748 +# github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8 ## explicit; go 1.20 github.com/basenana/friday/config github.com/basenana/friday/pkg/build/withvector