Merge pull request #84 from basenana/fix/summary

skip summary if content has less than 300 token
basenana · Dec 16, 2023 · 37b58cc · 37b58cc
2 parents 00aeb13 + d508e05
commit 37b58cc
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 14 deletions.
diff --git a/go.mod b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.26
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0
 	github.com/aws/smithy-go v1.13.5
-	github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748
+	github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8
 	github.com/blevesearch/bleve/v2 v2.3.10
 	github.com/bluele/gcache v0.0.2
 	github.com/getsentry/sentry-go v0.22.0

diff --git a/go.sum b/go.sum
@@ -59,6 +59,8 @@ github.com/basenana/friday v0.0.0-20231207142041-8f7eb86afc78 h1:PPj10ZL8OUIzT2w
 github.com/basenana/friday v0.0.0-20231207142041-8f7eb86afc78/go.mod h1:OGCCDNAXqN/ZTqO8EsaGGUBt5GrVwb/BVdnDQA4KbZI=
 github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748 h1:STmXI0Q6G3P55zZJRpkgJKQrcyb3/bTgKD/f2QJSU2Y=
 github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748/go.mod h1:OGCCDNAXqN/ZTqO8EsaGGUBt5GrVwb/BVdnDQA4KbZI=
+github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8 h1:jGuzKXoNJ4lIHe/SJmCfMZWSmo3DdDu7evZWNmxXkcs=
+github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8/go.mod h1:OGCCDNAXqN/ZTqO8EsaGGUBt5GrVwb/BVdnDQA4KbZI=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=

diff --git a/pkg/plugin/buildin/summary.go b/pkg/plugin/buildin/summary.go
@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/basenana/friday/pkg/utils/files"
 	"go.uber.org/zap"
 
 	"github.com/basenana/nanafs/pkg/friday"
@@ -33,6 +34,7 @@ import (
 const (
 	SummaryPluginName    = "summary"
 	SummaryPluginVersion = "1.0"
+	DefaultSummaryToken  = 300
 )
 
 type SummaryPlugin struct {
@@ -84,7 +86,12 @@ func (i *SummaryPlugin) Run(ctx context.Context, request *pluginapi.Request) (*p
 	}
 
 	trimmedContent := utils.ContentTrim(docType, buf.String())
-	i.logger(ctx).Infow("get docs", "length", len(trimmedContent), "entryId", request.EntryId)
+	length := files.Length(trimmedContent)
+	if length <= DefaultSummaryToken {
+		i.logger(ctx).Infow("skip summary, length less than default summary token.", "length", length, "default token", DefaultSummaryToken, "entryId", request.EntryId)
+		return pluginapi.NewResponseWithResult(map[string]any{}), nil
+	}
+	i.logger(ctx).Infow("get docs", "length", length, "entryId", request.EntryId)
 	summary, usage, err := friday.SummaryFile(ctx, fmt.Sprintf("entry_%d", request.EntryId), trimmedContent)
 	if err != nil {
 		return pluginapi.NewFailedResponse(fmt.Sprintf("summary documents failed: %s", err)), nil

diff --git a/vendor/github.com/basenana/friday/pkg/vectorstore/postgres/migrate.go b/vendor/github.com/basenana/friday/pkg/vectorstore/postgres/migrate.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -128,7 +128,7 @@ github.com/aws/smithy-go/waiter
 ## explicit
 github.com/aymerick/douceur/css
 github.com/aymerick/douceur/parser
-# github.com/basenana/friday v0.0.0-20231209065908-7ba12e67f748
+# github.com/basenana/friday v0.0.0-20231216063336-c2c9735897d8
 ## explicit; go 1.20
 github.com/basenana/friday/config
 github.com/basenana/friday/pkg/build/withvector