Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix motion photo files with MP~2 extension marked unsupported and skipped #405 #419

Merged
merged 11 commits into from
Aug 1, 2024
Merged

fix motion photo files with MP~2 extension marked unsupported and skipped #405 #419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
7e8875d
Improve get debug data
simulot Jul 30, 2024
31cbfae
adapt the fakefs reader
simulot Jul 30, 2024
095f0cb
fix duplicate counts when same path in the archive
simulot Jul 31, 2024
a0a1ab0
fix Duplicated assets already uploaded are not counted as Duplicates …
simulot Jul 31, 2024
9602c74
Merge branch 'main' into fix-0.21
simulot Jul 31, 2024
846e138
fix Movie part of live pictures already present on the server aren't …
simulot Jul 31, 2024
555be96
fix upload counters for live photos
simulot Jul 31, 2024
58febb9
add upload of .MP~* motion pictures
simulot Aug 1, 2024
9a8d262
add support of .MP~* in google photos
simulot Aug 1, 2024
82b7e65
fix motion photo files with MP~2 extension marked unsupported and ski…
simulot Aug 1, 2024
c9952f0
linter messages
simulot Aug 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 101 additions & 45 deletions browser/files/localassets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ type fileLinks struct {
type LocalAssetBrowser struct {
fsyss []fs.FS
albums map[string]string
catalogs map[fs.FS]map[string]map[string]fileLinks // per FS, DIR and base name
catalogs map[fs.FS]map[string][]string
log *fileevent.Recorder
sm immich.SupportedMedia
bannedFiles namematcher.List // list of file pattern to be exclude
Expand All @@ -38,9 +38,10 @@ func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, fsyss ...fs.FS) (
return &LocalAssetBrowser{
fsyss: fsyss,
albums: map[string]string{},
catalogs: map[fs.FS]map[string]map[string]fileLinks{},
catalogs: map[fs.FS]map[string][]string{},
log: l,
whenNoDate: "FILE",
sm: immich.DefaultSupportedMedia,
}, nil
}

Expand Down Expand Up @@ -71,17 +72,16 @@ func (la *LocalAssetBrowser) Prepare(ctx context.Context) error {
}

func (la *LocalAssetBrowser) passOneFsWalk(ctx context.Context, fsys fs.FS) error {
fsCatalog := map[string]map[string]fileLinks{}
la.catalogs[fsys] = map[string][]string{}
err := fs.WalkDir(fsys, ".",
func(name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

if d.IsDir() {
fsCatalog[name] = map[string]fileLinks{}
return nil
la.catalogs[fsys][name] = []string{}
}

select {
case <-ctx.Done():
// If the context has been cancelled, return immediately
Expand All @@ -100,40 +100,25 @@ func (la *LocalAssetBrowser) passOneFsWalk(ctx context.Context, fsys fs.FS) erro
return nil
}

linkBase := strings.TrimSuffix(base, ext)
for {
e := path.Ext(linkBase)
if la.sm.IsMedia(e) {
linkBase = strings.TrimSuffix(linkBase, e)
continue
}
break
}
dirLinks := fsCatalog[dir]
links := dirLinks[linkBase]
cat := la.catalogs[fsys][dir]

switch mediaType {
case immich.TypeImage:
links.image = name
la.log.Record(ctx, fileevent.DiscoveredImage, nil, name)
case immich.TypeVideo:
links.video = name
la.log.Record(ctx, fileevent.DiscoveredVideo, nil, name)
case immich.TypeSidecar:
links.sidecar = name
la.log.Record(ctx, fileevent.DiscoveredSidecar, nil, name)
}

if la.bannedFiles.Match(name) {
la.log.Record(ctx, fileevent.DiscoveredDiscarded, nil, name, "reason", "banned file")
return nil
}
dirLinks[linkBase] = links
fsCatalog[dir] = dirLinks
la.catalogs[fsys][dir] = append(cat, name)
}
return nil
})
la.catalogs[fsys] = fsCatalog
return err
}

Expand All @@ -150,46 +135,117 @@ func (la *LocalAssetBrowser) Browse(ctx context.Context) chan *browser.LocalAsse
}
}
for _, fsys := range la.fsyss {
dirLinks := la.catalogs[fsys]
dirKeys := gen.MapKeys(dirLinks)
sort.Strings(dirKeys)
for _, d := range dirKeys {
linksList := la.catalogs[fsys][d]
linksKeys := gen.MapKeys(linksList)
sort.Strings(linksKeys)
for _, l := range linksKeys {
dirs := gen.MapKeys(la.catalogs[fsys])
sort.Strings(dirs)
for _, dir := range dirs {
links := map[string]fileLinks{}
files := la.catalogs[fsys][dir]

if len(files) == 0 {
continue
}

// Scan images first
for _, file := range files {
ext := path.Ext(file)
if la.sm.TypeFromExt(ext) == immich.TypeImage {
linked := links[file]
linked.image = file
links[file] = linked
}
}

next:
for _, file := range files {
ext := path.Ext(file)
t := la.sm.TypeFromExt(ext)
if t == immich.TypeImage {
continue next
}

base := strings.TrimSuffix(file, ext)
switch t {
case immich.TypeSidecar:
if image, ok := links[base]; ok {
// file.ext.XMP -> file.ext
image.sidecar = file
links[base] = image
continue next
}
for f := range links {
if strings.TrimSuffix(f, path.Ext(f)) == base {
if image, ok := links[f]; ok {
// base.XMP -> base.ext
image.sidecar = file
links[f] = image
continue next
}
}
}
case immich.TypeVideo:
if image, ok := links[base]; ok {
// file.MP.ext -> file.ext
image.sidecar = file
links[base] = image
continue next
}
for f := range links {
if strings.TrimSuffix(f, path.Ext(f)) == base {
if image, ok := links[f]; ok {
// base.MP4 -> base.ext
image.video = file
links[f] = image
continue next
}
}
if strings.TrimSuffix(f, path.Ext(f)) == file {
if image, ok := links[f]; ok {
// base.MP4 -> base.ext
image.video = file
links[f] = image
continue next
}
}
}
// Unlinked video
links[file] = fileLinks{video: file}
}
}

files = gen.MapKeys(links)
sort.Strings(files)
for _, file := range files {
var a *browser.LocalAssetFile
links := linksList[l]
linked := links[file]

if links.image != "" {
a, err = la.assetFromFile(fsys, links.image)
if linked.image != "" {
a, err = la.assetFromFile(fsys, linked.image)
if err != nil {
errFn(links.image, err)
errFn(linked.image, err)
return
}
if links.video != "" {
a.LivePhoto, err = la.assetFromFile(fsys, links.video)
if linked.video != "" {
a.LivePhoto, err = la.assetFromFile(fsys, linked.video)
if err != nil {
errFn(links.video, err)
errFn(linked.video, err)
return
}
}
} else if links.video != "" {
a, err = la.assetFromFile(fsys, links.video)
} else if linked.video != "" {
a, err = la.assetFromFile(fsys, linked.video)
if err != nil {
errFn(links.video, err)
errFn(linked.video, err)
return
}
}

if a != nil && links.sidecar != "" {
if a != nil && linked.sidecar != "" {
a.SideCar = metadata.SideCarFile{
FSys: fsys,
FileName: links.sidecar,
FileName: linked.sidecar,
}
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, links.sidecar, "main", a.FileName)
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, nil, linked.sidecar, "main", a.FileName)
}

select {
case <-ctx.Done():
return
Expand Down
112 changes: 75 additions & 37 deletions browser/files/localassets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
package files_test
package files

import (
"context"
"errors"
"io/fs"
"path"
"reflect"
"sort"
"testing"

"github.com/kr/pretty"
"github.com/psanford/memfs"
"github.com/simulot/immich-go/browser/files"
"github.com/simulot/immich-go/helpers/fileevent"
"github.com/simulot/immich-go/helpers/namematcher"
"github.com/simulot/immich-go/immich"
Expand All @@ -37,49 +36,77 @@ func (mfs *inMemFS) addFile(name string) *inMemFS {
return mfs
}

func generateFS() *inMemFS {
return newInMemFS().
addFile("root_01.jpg").
addFile("photos/photo_01.jpg").
addFile("photos/photo_02.cr3").
addFile("photos/photo_03.jpg").
addFile("photos/summer 2023/20230801-001.jpg").
addFile("photos/summer 2023/20230801-002.jpg").
addFile("photos/summer 2023/20230801-003.cr3").
addFile("@eaDir/thb1.jpg").
addFile("photos/SYNOFILE_THUMB_0001.jpg").
addFile("photos/summer 2023/.@__thumb/thb2.jpg")
}

func TestLocalAssets(t *testing.T) {
tc := []struct {
name string
expected []string
fsys fs.FS
expected map[string]fileLinks
}{
{
name: "all",
expected: []string{
"root_01.jpg",
"photos/photo_01.jpg",
"photos/photo_02.cr3",
"photos/photo_03.jpg",
"photos/summer 2023/20230801-001.jpg",
"photos/summer 2023/20230801-002.jpg",
"photos/summer 2023/20230801-003.cr3",
name: "simple",
fsys: newInMemFS().
addFile("root_01.jpg").
addFile("photos/photo_01.jpg").
addFile("photos/photo_02.cr3").
addFile("photos/photo_03.jpg").
addFile("photos/summer 2023/20230801-001.jpg").
addFile("photos/summer 2023/20230801-002.jpg").
addFile("photos/summer 2023/20230801-003.cr3").
addFile("@eaDir/thb1.jpg").
addFile("photos/SYNOFILE_THUMB_0001.jpg").
addFile("photos/summer 2023/.@__thumb/thb2.jpg"),
expected: map[string]fileLinks{
"root_01.jpg": {image: "root_01.jpg"},
"photos/photo_01.jpg": {image: "photos/photo_01.jpg"},
"photos/photo_02.cr3": {image: "photos/photo_02.cr3"},
"photos/photo_03.jpg": {image: "photos/photo_03.jpg"},
"photos/summer 2023/20230801-001.jpg": {image: "photos/summer 2023/20230801-001.jpg"},
"photos/summer 2023/20230801-002.jpg": {image: "photos/summer 2023/20230801-002.jpg"},
"photos/summer 2023/20230801-003.cr3": {image: "photos/summer 2023/20230801-003.cr3"},
},
},
{
name: "motion picture",
fsys: newInMemFS().
addFile("motion/PXL_20210102_221126856.MP~2").
addFile("motion/PXL_20210102_221126856.MP~2.jpg").
addFile("motion/PXL_20210102_221126856.MP.jpg").
addFile("motion/PXL_20210102_221126856.MP").
addFile("motion/20231227_152817.jpg").
addFile("motion/20231227_152817.MP4"),
expected: map[string]fileLinks{
"motion/PXL_20210102_221126856.MP.jpg": {image: "motion/PXL_20210102_221126856.MP.jpg", video: "motion/PXL_20210102_221126856.MP"},
"motion/PXL_20210102_221126856.MP~2.jpg": {image: "motion/PXL_20210102_221126856.MP~2.jpg", video: "motion/PXL_20210102_221126856.MP~2"},
"motion/20231227_152817.jpg": {image: "motion/20231227_152817.jpg", video: "motion/20231227_152817.MP4"},
},
},
{
name: "sidecar",
fsys: newInMemFS().
addFile("root_01.jpg").
addFile("root_01.XMP").
addFile("root_02.jpg").
addFile("root_02.jpg.XMP").
addFile("video_01.mp4").
addFile("video_01.mp4.XMP").
addFile("root_03.MP.jpg").
addFile("root_03.MP.jpg.XMP").
addFile("root_03.MP"),
expected: map[string]fileLinks{
"root_01.jpg": {image: "root_01.jpg", sidecar: "root_01.XMP"},
"root_02.jpg": {image: "root_02.jpg", sidecar: "root_02.jpg.XMP"},
"root_03.MP.jpg": {image: "root_03.MP.jpg", sidecar: "root_03.MP.jpg.XMP", video: "root_03.MP"},
"video_01.mp4": {video: "video_01.mp4", sidecar: "video_01.mp4.XMP"},
},
},
}

for _, c := range tc {
t.Run(c.name, func(t *testing.T) {
fsys := generateFS()
if fsys.err != nil {
t.Error(fsys.err)
return
}
fsys := c.fsys
ctx := context.Background()

b, err := files.NewLocalFiles(ctx, fileevent.NewRecorder(nil, false), fsys)
b, err := NewLocalFiles(ctx, fileevent.NewRecorder(nil, false), fsys)
if err != nil {
t.Error(err)
}
Expand All @@ -96,12 +123,23 @@ func TestLocalAssets(t *testing.T) {
t.Error(err)
}

results := []string{}
results := map[string]fileLinks{}
for a := range b.Browse(ctx) {
results = append(results, a.FileName)
links := fileLinks{}
ext := path.Ext(a.FileName)
if b.sm.TypeFromExt(ext) == immich.TypeImage {
links.image = a.FileName
if a.LivePhoto != nil {
links.video = a.LivePhoto.FileName
}
} else {
links.video = a.FileName
}
if a.SideCar.FileName != "" {
links.sidecar = a.SideCar.FileName
}
results[a.FileName] = links
}
sort.Strings(c.expected)
sort.Strings(results)

if !reflect.DeepEqual(results, c.expected) {
t.Errorf("difference\n")
Expand Down
Loading