SYNOFILE_THUMB_{some pattern}.jpg should be automatically skipped fro…

…m being uploaded (#366)

Fixes #319

browser/files/localassets.go

@@ -13,6 +13,7 @@ import (
 	"github.com/simulot/immich-go/helpers/fileevent"
 	"github.com/simulot/immich-go/helpers/fshelper"
 	"github.com/simulot/immich-go/helpers/gen"
+	"github.com/simulot/immich-go/helpers/namematcher"
 	"github.com/simulot/immich-go/immich"
 	"github.com/simulot/immich-go/immich/metadata"
 )
@@ -24,12 +25,13 @@ type fileLinks struct {
 }
 
 type LocalAssetBrowser struct {
-	fsyss      []fs.FS
-	albums     map[string]string
-	catalogs   map[fs.FS]map[string]map[string]fileLinks // per FS, DIR and base name
-	log        *fileevent.Recorder
-	sm         immich.SupportedMedia
-	whenNoDate string
+	fsyss       []fs.FS
+	albums      map[string]string
+	catalogs    map[fs.FS]map[string]map[string]fileLinks // per FS, DIR and base name
+	log         *fileevent.Recorder
+	sm          immich.SupportedMedia
+	bannedFiles namematcher.List // list of file pattern to be exclude
+	whenNoDate  string
 }
 
 func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
@@ -47,6 +49,11 @@ func (la *LocalAssetBrowser) SetSupportedMedia(sm immich.SupportedMedia) *LocalA
 	return la
 }
 
+func (la *LocalAssetBrowser) SetBannedFiles(banned namematcher.List) *LocalAssetBrowser {
+	la.bannedFiles = banned
+	return la
+}
+
 func (la *LocalAssetBrowser) SetWhenNoDate(opt string) *LocalAssetBrowser {
 	la.whenNoDate = opt
 	return la
@@ -116,6 +123,11 @@ func (la *LocalAssetBrowser) passOneFsWalk(ctx context.Context, fsys fs.FS) erro
 					links.sidecar = name
 					la.log.Record(ctx, fileevent.DiscoveredSidecar, nil, name)
 				}
+
+				if la.bannedFiles.Match(name) {
+					la.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, name, "reason", "banned file")
+					return nil
+				}
 				dirLinks[linkBase] = links
 				fsCatalog[dir] = dirLinks
 			}

browser/files/localassets_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/psanford/memfs"
 	"github.com/simulot/immich-go/browser/files"
 	"github.com/simulot/immich-go/helpers/fileevent"
+	"github.com/simulot/immich-go/helpers/namematcher"
 	"github.com/simulot/immich-go/immich"
 )
 
@@ -44,7 +45,10 @@ func generateFS() *inMemFS {
 		addFile("photos/photo_03.jpg").
 		addFile("photos/summer 2023/20230801-001.jpg").
 		addFile("photos/summer 2023/20230801-002.jpg").
-		addFile("photos/summer 2023/20230801-003.cr3")
+		addFile("photos/summer 2023/20230801-003.cr3").
+		addFile("@eaDir/thb1.jpg").
+		addFile("photos/SYNOFILE_THUMB_0001.jpg").
+		addFile("photos/summer 2023/.@__thumb/thb2.jpg")
 }
 
 func TestLocalAssets(t *testing.T) {
@@ -79,6 +83,11 @@ func TestLocalAssets(t *testing.T) {
 			if err != nil {
 				t.Error(err)
 			}
+			l, err := namematcher.New(`@eaDir/`, `.@__thumb`, `SYNOFILE_THUMB_*.*`)
+			if err != nil {
+				t.Error(err)
+			}
+			b.SetBannedFiles(l)
 			b.SetSupportedMedia(immich.DefaultSupportedMedia)
 			b.SetWhenNoDate("FILE")
 

browser/gp/googlephotos.go

@@ -14,6 +14,7 @@ import (
 	"github.com/simulot/immich-go/helpers/fileevent"
 	"github.com/simulot/immich-go/helpers/fshelper"
 	"github.com/simulot/immich-go/helpers/gen"
+	"github.com/simulot/immich-go/helpers/namematcher"
 	"github.com/simulot/immich-go/immich"
 	"github.com/simulot/immich-go/immich/metadata"
 )
@@ -26,6 +27,7 @@ type Takeout struct {
 	albums     map[string]browser.LocalAlbum // tack album names by folder
 	log        *fileevent.Recorder
 	sm         immich.SupportedMedia
+	banned     namematcher.List // Banned files
 }
 
 // dirCatalog collects all directory catalogs
@@ -71,6 +73,11 @@ func NewTakeout(ctx context.Context, l *fileevent.Recorder, sm immich.SupportedM
 	return &to, nil
 }
 
+func (to *Takeout) SetBannedFiles(banned namematcher.List) *Takeout {
+	to.banned = banned
+	return to
+}
+
 // Prepare scans all files in all walker to build the file catalog of the archive
 // metadata files content is read and kept
 
@@ -158,6 +165,11 @@ func (to *Takeout) passOneFsWalk(ctx context.Context, w fs.FS) error {
 				case immich.TypeImage:
 					to.log.Record(ctx, fileevent.DiscoveredImage, nil, name)
 				}
+
+				if to.banned.Match(name) {
+					to.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, name, "reason", "banned file")
+					return nil
+				}
 				dirCatalog.unMatchedFiles[base] = fileInfo{
 					base:   base,
 					fsys:   w,

cmd/upload/e2e_upload_folder_test.go

@@ -530,6 +530,23 @@ func Test_EnrichedAlbum_297(t *testing.T) {
 	runCase(t, tc)
 }
 
+func Test_BannedFiles_(t *testing.T) {
+	initMyEnv(t)
+
+	tc := testCase{
+		name: "Test_BannedFiles_",
+		args: []string{
+			"-exclude-files=backup/",
+			"-exclude-files=copy).*",
+			"TEST_DATA/banned",
+		},
+		resetImmich: true,
+		expectError: false,
+		APITrace:    false,
+	}
+	runCase(t, tc)
+}
+
 // Check if the small version of the photos loaded with the takeout
 // is replaced by the better version
 func Test_SmallTakeout_Better_p1(t *testing.T) {

cmd/upload/upload.go

@@ -25,6 +25,7 @@ import (
 	"github.com/simulot/immich-go/helpers/fshelper"
 	"github.com/simulot/immich-go/helpers/gen"
 	"github.com/simulot/immich-go/helpers/myflag"
+	"github.com/simulot/immich-go/helpers/namematcher"
 	"github.com/simulot/immich-go/helpers/stacking"
 	"github.com/simulot/immich-go/immich"
 	"golang.org/x/sync/errgroup"
@@ -56,6 +57,7 @@ type UpCmd struct {
 	StackBurst             bool             // Stack burst (Default: TRUE)
 	DiscardArchived        bool             // Don't import archived assets (Default: FALSE)
 	WhenNoDate             string           // When the date can't be determined use the FILE's date or NOW (default: FILE)
+	BannedFiles            namematcher.List // List of banned file name patterns
 
 	BrowserConfig Configuration
 
@@ -87,6 +89,16 @@ func newCommand(ctx context.Context, common *cmd.SharedFlags, args []string) (*U
 	app := UpCmd{
 		SharedFlags: common,
 	}
+	app.BannedFiles, err = namematcher.New(
+		`@eaDir/`,
+		`@__thumb/`,          // QNAP
+		`SYNOFILE_THUMB_*.*`, // SYNOLOGY
+		`Lightroom Catalog/`, // LR
+		`thumbnails/`,        // Android photo
+	)
+	if err != nil {
+		return nil, err
+	}
 
 	app.SharedFlags.SetFlags(cmd)
 	cmd.BoolFunc(
@@ -156,6 +168,9 @@ func newCommand(ctx context.Context, common *cmd.SharedFlags, args []string) (*U
 		"when-no-date",
 		"FILE",
 		" When the date of take can't be determined, use the FILE's date or the current time NOW. (default: FILE)")
+
+	cmd.Var(&app.BannedFiles, "exclude-files", "Ignore files based on a pattern. Case insensitive. Add one option for each pattern do you need.")
+
 	err = cmd.Parse(args)
 	if err != nil {
 		return nil, err
@@ -762,7 +777,12 @@ func (app *UpCmd) isInAlbum(a *browser.LocalAssetFile, album string) bool {
 
 func (app *UpCmd) ReadGoogleTakeOut(ctx context.Context, fsyss []fs.FS) (browser.Browser, error) {
 	app.Delete = false
-	return gp.NewTakeout(ctx, app.Jnl, app.Immich.SupportedMedia(), fsyss...)
+	b, err := gp.NewTakeout(ctx, app.Jnl, app.Immich.SupportedMedia(), fsyss...)
+	if err != nil {
+		return nil, err
+	}
+	b.SetBannedFiles(app.BannedFiles)
+	return b, err
 }
 
 func (app *UpCmd) ExploreLocalFolder(ctx context.Context, fsyss []fs.FS) (browser.Browser, error) {
@@ -772,6 +792,7 @@ func (app *UpCmd) ExploreLocalFolder(ctx context.Context, fsyss []fs.FS) (browse
 	}
 	b.SetSupportedMedia(app.Immich.SupportedMedia())
 	b.SetWhenNoDate(app.WhenNoDate)
+	b.SetBannedFiles(app.BannedFiles)
 	return b, nil
 }
 

docs/releases.md

@@ -5,6 +5,20 @@
 - [Github Sponsor page](https://github.com/sponsors/simulot)
 - [paypal donor page](https://www.paypal.com/donate/?hosted_button_id=VGU2SQE88T2T4)
 
+### feat: exclude files based on a pattern
+
+Use the `-exclude-files=PATTERN` to exclude certain files or directories from the upload. Repeat the option for each pattern do you need. The following directories are excluded automatically:
+- @eaDir/
+- @__thumb/
+- SYNOFILE_THUMB_\*.\*
+- Lightroom Catalog/
+- thumbnails/
+
+Example, the following command excludes any files in directories called backup or draft and any file with name finishing with "copy)" as PXL_20231006_063121958 (another copy).jpg:
+```sh
+immich-go -sever=xxxxx -key=yyyyy upload -exclude-files=backup/ -exclude-files=draft/ -exclude=copy).*  /path/to/your/files
+```
+
 ## Release 0.19.1
 
 ### fix: UploadAsset

helpers/namematcher/list.go

@@ -0,0 +1,138 @@
+package namematcher
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// List of file patterns used to ban unwanted files
+// Pattern can be a part of the path, a file name..
+
+type List struct {
+	re       []*regexp.Regexp
+	patterns []string
+}
+
+func New(patterns ...string) (List, error) {
+	l := List{}
+	for _, name := range patterns {
+		err := l.Set(name)
+		if err != nil {
+			return List{}, err
+		}
+	}
+	return l, nil
+}
+
+func (l List) Match(name string) bool {
+	for _, re := range l.re {
+		if re.MatchString(name) {
+			return true
+		}
+	}
+	return false
+}
+
+func fetchRune(b []byte) ([]byte, rune) {
+	r, size := utf8.DecodeRune(b)
+	b = b[size:]
+	return b, r
+}
+
+// transform a glob styled pattern into a regular expression
+func patternToRe(pattern string) (*regexp.Regexp, error) {
+	var r strings.Builder
+	var inBrackets bool
+	var b rune
+	buf := []byte(pattern)
+
+	r.WriteString("(?i)") // make the pattern case insensitive
+
+	for len(buf) > 0 {
+		buf, b = fetchRune(buf)
+		switch b {
+		case '*':
+			r.WriteString(`[^./]*`)
+		case '?':
+			r.WriteString(`[^./]`)
+		case '.', '^', '$', '(', ')', '|':
+			r.WriteRune('\\')
+			r.WriteRune(b)
+		case '\\':
+			r.WriteRune(b)
+			buf, b = fetchRune(buf)
+			r.WriteRune(b)
+		case '[':
+			inBrackets = true
+			r.WriteRune(b)
+		brackets:
+			for len(buf) > 0 {
+				buf, b = fetchRune(buf)
+				switch b {
+				case ']':
+					inBrackets = false
+					r.WriteRune(b)
+					break brackets
+				default:
+					lCase, uCase := unicode.ToLower(b), unicode.ToUpper(b)
+					r.WriteRune(lCase)
+					if lCase != uCase {
+						r.WriteRune(uCase)
+					}
+				}
+			}
+		default:
+			r.WriteRune(b)
+		}
+	}
+	if inBrackets {
+		return nil, fmt.Errorf("invalid file name pattern: %s", pattern)
+	}
+	re, err := regexp.Compile(r.String())
+	if err != nil {
+		return nil, fmt.Errorf("invalid file name pattern: %s", pattern)
+	}
+	return re, nil
+}
+
+/*
+	Implements the flag.Value interface for the list of banned files
+	Check the validity of the pattern
+*/
+
+func (l *List) Set(s string) error {
+	if l == nil {
+		return errors.New("namematcher  list not initialized")
+	}
+	if s == "" {
+		return nil
+	}
+	re, err := patternToRe(s)
+	if err != nil {
+		return err
+	}
+	l.re = append(l.re, re)
+	l.patterns = append(l.patterns, s)
+	return nil
+}
+
+func (l List) String() string {
+	var s strings.Builder
+	for i, pattern := range l.patterns {
+		if i > 0 {
+			s.WriteString(", ")
+		}
+		s.WriteRune('\'')
+		s.WriteString(pattern)
+		s.WriteRune('\'')
+	}
+	return s.String()
+}
+
+func (l *List) Get() any {
+	return *l
+}