Merge branch 'merge/fix-pipelines-and-linter' into 'master'

Merge: Fix pipelines and linter See merge request logoper/doc-notifier!11
breadrock1 · Mar 27, 2024 · ad65c51 · ad65c51
2 parents 275ed6a + 61e8354
commit ad65c51
Show file tree

Hide file tree

Showing 20 changed files with 58 additions and 53 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -25,4 +25,4 @@ lint:
 test:
   stage: test
   script:
-    - for test_module in $(git ls-files test); do go test -v -count=1 -race $test_module; done
+    - for test_module in $(git ls-files tests); do go test -v -count=1 -race $test_module; done
diff --git a/.golangci.yml b/.golangci.yml
@@ -11,12 +11,13 @@ linters-settings:
 
 issues:
   exclude-rules:
-    - path: _test\.go
+    - path: _test.go
       linters:
         - errcheck
         - dupl
         - gocyclo
         - gosec
+        - gocritic
     - path: document.go
       linters:
         - gosec
@@ -27,6 +28,12 @@ issues:
       linters:
         - revive
         - ineffassign
+    - path: langchain.go
+      linters:
+        - dupl
+    - path: assistant.go
+      linters:
+        - dupl
 
 linters:
   disable-all: true

diff --git a/internal/pkg/ocr/assistant/assistant.go b/internal/pkg/ocr/assistant/assistant.go
@@ -11,12 +11,12 @@ import (
 	"path/filepath"
 )
 
-type AssistantOCR struct {
+type Service struct {
 	address string
 }
 
-func New(address string) *AssistantOCR {
-	return &AssistantOCR{
+func New(address string) *Service {
+	return &Service{
 		address: address,
 	}
 }
@@ -27,7 +27,7 @@ type DocumentForm struct {
 	Context string `json:"context"`
 }
 
-func (ro *AssistantOCR) RecognizeFile(filePath string) (string, error) {
+func (ro *Service) RecognizeFile(filePath string) (string, error) {
 	fileHandle, err := os.Open(filePath)
 	if err != nil {
 		log.Println("Failed while opening file: ", err)
@@ -67,7 +67,7 @@ func (ro *AssistantOCR) RecognizeFile(filePath string) (string, error) {
 	return resTest.Context, nil
 }
 
-func (ro *AssistantOCR) RecognizeFileData(data []byte) (string, error) {
+func (ro *Service) RecognizeFileData(data []byte) (string, error) {
 	var reqBody bytes.Buffer
 	writer := multipart.NewWriter(&reqBody)
 

diff --git a/internal/pkg/ocr/dedoc/dedoc.go b/internal/pkg/ocr/dedoc/dedoc.go
@@ -11,12 +11,12 @@ import (
 	"path/filepath"
 )
 
-type DedocOCR struct {
+type Service struct {
 	address string
 }
 
-func New(address string) *DedocOCR {
-	return &DedocOCR{
+func New(address string) *Service {
+	return &Service{
 		address: address,
 	}
 }
@@ -27,7 +27,7 @@ type DocumentForm struct {
 	Context string `json:"text"`
 }
 
-func (do *DedocOCR) RecognizeFile(filePath string) (string, error) {
+func (do *Service) RecognizeFile(filePath string) (string, error) {
 	fileHandle, err := os.Open(filePath)
 	if err != nil {
 		log.Println("Failed while opening file: ", err)
@@ -67,7 +67,7 @@ func (do *DedocOCR) RecognizeFile(filePath string) (string, error) {
 	return resTest.Context, nil
 }
 
-func (do *DedocOCR) RecognizeFileData(data []byte) (string, error) {
+func (do *Service) RecognizeFileData(data []byte) (string, error) {
 	var reqBody bytes.Buffer
 	writer := multipart.NewWriter(&reqBody)
 

diff --git a/internal/pkg/ocr/ocr.go b/internal/pkg/ocr/ocr.go
@@ -6,7 +6,7 @@ import (
 	"doc-notifier/internal/pkg/ocr/raw"
 )
 
-type OcrService struct {
+type Service struct {
 	Ocr Recognizer
 }
 
@@ -15,8 +15,8 @@ type Recognizer interface {
 	RecognizeFileData(data []byte) (string, error)
 }
 
-func New(options *Options) *OcrService {
-	service := &OcrService{}
+func New(options *Options) *Service {
+	service := &Service{}
 
 	switch options.Mode {
 	case ReadRawFile:

diff --git a/internal/pkg/ocr/raw/raw.go b/internal/pkg/ocr/raw/raw.go
@@ -5,14 +5,14 @@ import (
 	"os"
 )
 
-type RawExractor struct {
+type Service struct {
 }
 
-func New() *RawExractor {
-	return &RawExractor{}
+func New() *Service {
+	return &Service{}
 }
 
-func (re *RawExractor) RecognizeFile(filePath string) (string, error) {
+func (re *Service) RecognizeFile(filePath string) (string, error) {
 	bytesData, err := os.ReadFile(filePath)
 	if err != nil {
 		log.Println("Failed while reading file: ", err)
@@ -22,6 +22,6 @@ func (re *RawExractor) RecognizeFile(filePath string) (string, error) {
 	return string(bytesData), nil
 }
 
-func (re *RawExractor) RecognizeFileData(data []byte) (string, error) {
+func (re *Service) RecognizeFileData(data []byte) (string, error) {
 	return string(data), nil
 }
diff --git a/internal/pkg/reader/document.go b/internal/pkg/reader/document.go
@@ -108,45 +108,45 @@ func extractApplicationMimeType(attribute string) string {
 	return "unknown"
 }
 
-func (f *ReaderService) SetContentData(document *Document, data string) {
+func (f *Service) SetContentData(document *Document, data string) {
 	document.Content = data
 }
 
-func (f *ReaderService) SetContentVector(document *Document, data []float64) {
+func (f *Service) SetContentVector(document *Document, data []float64) {
 	document.ContentVector = data
 }
 
-func (f *ReaderService) AppendContentVector(document *Document, data []float64) {
+func (f *Service) AppendContentVector(document *Document, data []float64) {
 	document.ContentVector = append(document.ContentVector, data...)
 }
 
-func (f *ReaderService) ComputeMd5Hash(document *Document) {
+func (f *Service) ComputeMd5Hash(document *Document) {
 	data := []byte(document.Content)
 	document.DocumentMD5 = fmt.Sprintf("%x", md5.Sum(data))
 }
 
-func (f *ReaderService) ComputeContentMd5Hash(document *Document) {
+func (f *Service) ComputeContentMd5Hash(document *Document) {
 	if len(document.DocumentMD5) == 0 {
 		f.ComputeMd5Hash(document)
 	}
 	document.ContentMD5 = document.DocumentMD5
 }
 
-func (f *ReaderService) ComputeSsdeepHash(document *Document) {
+func (f *Service) ComputeSsdeepHash(document *Document) {
 	data := []byte(document.Content)
 	if hashData, err := ssdeep.FuzzyBytes(data); err == nil {
 		document.DocumentSSDEEP = hashData
 	}
 }
 
-func (f *ReaderService) ComputeUUID(document *Document) {
+func (f *Service) ComputeUUID(document *Document) {
 	data := []byte(document.Content)
 	if uuidToken, err := uuid.FromBytes(data); err == nil {
 		document.ContentUUID = uuidToken.String()
 	}
 }
 
-func (f *ReaderService) SplitContent(content string, chunkSize int) []string {
+func (f *Service) SplitContent(content string, chunkSize int) []string {
 	strLength := len(content)
 	splitLength := int(math.Ceil(float64(strLength) / float64(chunkSize)))
 	splitString := make([]string, splitLength)

diff --git a/internal/pkg/reader/reader.go b/internal/pkg/reader/reader.go
@@ -1,8 +1,8 @@
 package reader
 
-type ReaderService struct {
+type Service struct {
 }
 
-func New() *ReaderService {
-	return &ReaderService{}
+func New() *Service {
+	return &Service{}
 }
diff --git a/internal/pkg/reader/walker.go b/internal/pkg/reader/walker.go
@@ -9,7 +9,7 @@ import (
 	"time"
 )
 
-func (f *ReaderService) ParseCaughtFiles(filePath string) []*Document {
+func (f *Service) ParseCaughtFiles(filePath string) []*Document {
 	mu := &sync.Mutex{}
 	var customList []*Document
 

diff --git a/internal/pkg/searcher/searcher.go b/internal/pkg/searcher/searcher.go
@@ -10,17 +10,17 @@ import (
 
 const SearcherURL = "/document/new"
 
-type SearcherService struct {
+type Service struct {
 	address string
 }
 
-func New(address string) *SearcherService {
-	return &SearcherService{
+func New(address string) *Service {
+	return &Service{
 		address: address,
 	}
 }
 
-func (ss *SearcherService) StoreDocument(document *reader.Document) error {
+func (ss *Service) StoreDocument(document *reader.Document) error {
 	jsonData, err := json.Marshal(document)
 	if err != nil {
 		log.Println("Failed while marshaling doc: ", err)

diff --git a/internal/pkg/sender/sender.go b/internal/pkg/sender/sender.go
@@ -34,7 +34,7 @@ func SendRequest(body *bytes.Buffer, url *string, formData string) ([]byte, erro
 	}
 
 	if response.StatusCode > 200 {
-		log.Printf("Non Ok response status %d: %s", response.Status, string(respData))
+		log.Printf("Non Ok response status %s: %s", response.Status, string(respData))
 		return nil, errors.New("non 200 response code status")
 	}
 

diff --git a/internal/pkg/tokenizer/assistant.go b/internal/pkg/tokenizer/assistant.go
@@ -34,7 +34,7 @@ func (at *AssistantTokenizer) TokenizeTextData(content string) (*ComputedTokens,
 	}
 
 	contentData := strings.ReplaceAll(content, "\n", " ")
-	textVectors := &TokenizerForm{
+	textVectors := &GetTokensForm{
 		Text:              contentData,
 		ChunkSize:         at.ChunkSize,
 		ChunkOverlap:      at.ChunkOverlap,

diff --git a/internal/pkg/tokenizer/form.go b/internal/pkg/tokenizer/form.go
@@ -1,6 +1,6 @@
 package tokenizer
 
-type TokenizerForm struct {
+type GetTokensForm struct {
 	Text              string `json:"text"`
 	ChunkSize         int    `json:"chunk_size"`
 	ChunkOverlap      int    `json:"chunk_overlap"`

diff --git a/internal/pkg/tokenizer/langchain.go b/internal/pkg/tokenizer/langchain.go
@@ -34,7 +34,7 @@ func (lt *LangChainTokenizer) TokenizeTextData(content string) (*ComputedTokens,
 	}
 
 	contentData := strings.ReplaceAll(content, "\n", " ")
-	textVectors := &TokenizerForm{
+	textVectors := &GetTokensForm{
 		Text:              contentData,
 		ChunkSize:         lt.ChunkSize,
 		ChunkOverlap:      lt.ChunkOverlap,

diff --git a/internal/pkg/tokenizer/none.go b/internal/pkg/tokenizer/none.go
@@ -7,7 +7,7 @@ func NewNone() *NoneTokenizer {
 	return &NoneTokenizer{}
 }
 
-func (nt *NoneTokenizer) TokenizeTextData(text string) (*ComputedTokens, error) {
+func (nt *NoneTokenizer) TokenizeTextData(_ string) (*ComputedTokens, error) {
 	return &ComputedTokens{
 		Chunks:      0,
 		ChunkedText: []string{},

diff --git a/internal/pkg/tokenizer/tokenizer.go b/internal/pkg/tokenizer/tokenizer.go
@@ -1,6 +1,6 @@
 package tokenizer
 
-type TokenizerService struct {
+type Service struct {
 	Tokenizer        Tokenizer
 	TokenizerOptions *Options
 }
@@ -9,8 +9,8 @@ type Tokenizer interface {
 	TokenizeTextData(text string) (*ComputedTokens, error)
 }
 
-func New(options *Options) *TokenizerService {
-	service := &TokenizerService{
+func New(options *Options) *Service {
+	service := &Service{
 		TokenizerOptions: options,
 	}
 

diff --git a/internal/pkg/watcher/watcher.go b/internal/pkg/watcher/watcher.go
@@ -21,10 +21,10 @@ type NotifyWatcher struct {
 	directories []string
 	watcher     *fsnotify.Watcher
 
-	ocr       *ocr.OcrService
-	reader    *reader.ReaderService
-	searcher  *searcher.SearcherService
-	tokenizer *tokenizer.TokenizerService
+	ocr       *ocr.Service
+	reader    *reader.Service
+	searcher  *searcher.Service
+	tokenizer *tokenizer.Service
 }
 
 func New(options *Options) *NotifyWatcher {
@@ -102,7 +102,6 @@ func (nw *NotifyWatcher) parseEventSlot() {
 
 	for {
 		select {
-
 		case err, ok := <-nw.watcher.Errors:
 			if !ok {
 				return
@@ -132,7 +131,6 @@ func (nw *NotifyWatcher) parseEventSlot() {
 			}
 
 			t.Reset(waitFor)
-
 		}
 	}
 }

diff --git a/main.go b/main.go
@@ -40,6 +40,6 @@ func main() {
 	defer httpServer.StopServer()
 
 	killSignal := make(chan os.Signal, 1)
-	signal.Notify(killSignal, syscall.SIGINT, syscall.SIGKILL, syscall.SIGABRT)
+	signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGABRT)
 	<-killSignal
 }
diff --git a/tests/mocked/mocked.go b/tests/mocked/mocked.go
@@ -47,7 +47,7 @@ func StoreDocument(c echo.Context) error {
 }
 
 func ComputeTokens(c echo.Context) error {
-	tokensForm := &tokenizer.TokenizerForm{}
+	tokensForm := &tokenizer.GetTokensForm{}
 	decoder := json.NewDecoder(c.Request().Body)
 	_ = decoder.Decode(tokensForm)
 

diff --git a/tests/reader/reader_test.go b/tests/reader/reader_test.go
@@ -9,7 +9,7 @@ import (
 const TestcaseDirPath = "../../testcases/"
 
 func TestParseCaughtFiles(t *testing.T) {
-	fileReader := &reader.ReaderService{}
+	fileReader := &reader.Service{}
 
 	t.Run("Parse file by path", func(t *testing.T) {
 		documents := fileReader.ParseCaughtFiles(TestcaseDirPath + "directory/test_file_1.txt")