diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts index 3f910fa1e3756..1959a230ad231 100644 --- a/e2e/src/api/specs/library.e2e-spec.ts +++ b/e2e/src/api/specs/library.e2e-spec.ts @@ -421,7 +421,7 @@ describe('/libraries', () => { const { status } = await request(app) .post(`/libraries/${library.id}/scan`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ refreshModifiedFiles: true }); + .send(); expect(status).toBe(204); await utils.waitForQueueFinish(admin.accessToken, 'library'); @@ -453,7 +453,7 @@ describe('/libraries', () => { const { status } = await request(app) .post(`/libraries/${library.id}/scan`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ refreshModifiedFiles: true }); + .send(); expect(status).toBe(204); await utils.waitForQueueFinish(admin.accessToken, 'library'); @@ -499,7 +499,7 @@ describe('/libraries', () => { expect(newAssets.items).toEqual([]); }); - it('should set an asset offline its file is not in any import path', async () => { + it('should set an asset offline if its file is not in any import path', async () => { utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`); const library = await utils.createLibrary(admin.accessToken, { @@ -577,7 +577,7 @@ describe('/libraries', () => { ]); }); - it('should not trash an online asset', async () => { + it('should not set an asset offline if its file exists, is in an import path, and not covered by an exclusion pattern', async () => { const library = await utils.createLibrary(admin.accessToken, { ownerId: admin.userId, importPaths: [`${testAssetDirInternal}/temp`], @@ -601,6 +601,195 @@ describe('/libraries', () => { expect(assets).toEqual(assetsBefore); }); + + it('should set an offline asset to online if its file exists, is in an import path, and not covered by an exclusion pattern', async () => { + utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`); + + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/offline`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`); + + { + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + } + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + expect(offlineAsset.isTrashed).toBe(true); + expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`); + expect(offlineAsset.isOffline).toBe(true); + + { + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true }); + expect(assets.count).toBe(1); + } + + utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`); + + { + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + } + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const backOnlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(backOnlineAsset.isTrashed).toBe(false); + expect(backOnlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`); + expect(backOnlineAsset.isOffline).toBe(false); + + { + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + expect(assets.count).toBe(1); + } + }); + + it('should not set an offline asset to online if its file exists, is not covered by an exclusion pattern, but is outside of all import paths', async () => { + utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`); + + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/offline`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`); + + { + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + } + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + { + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true }); + expect(assets.count).toBe(1); + } + + const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(offlineAsset.isTrashed).toBe(true); + expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`); + expect(offlineAsset.isOffline).toBe(true); + + utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`); + + utils.createDirectory(`${testAssetDir}/temp/another-path/`); + + await utils.updateLibrary(admin.accessToken, library.id, { + importPaths: [`${testAssetDirInternal}/temp/another-path`], + }); + + { + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + } + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(stillOfflineAsset.isTrashed).toBe(true); + expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`); + expect(stillOfflineAsset.isOffline).toBe(true); + + { + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true }); + expect(assets.count).toBe(1); + } + + utils.removeDirectory(`${testAssetDir}/temp/another-path/`); + }); + + it('should not set an offline asset to online if its file exists, is in an import path, but is covered by an exclusion pattern', async () => { + utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`); + + const library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/offline`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id }); + + utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`); + + { + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + } + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + { + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true }); + expect(assets.count).toBe(1); + } + + const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(offlineAsset.isTrashed).toBe(true); + expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`); + expect(offlineAsset.isOffline).toBe(true); + + utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`); + + await utils.updateLibrary(admin.accessToken, library.id, { exclusionPatterns: ['**/offline/**'] }); + + { + const { status } = await request(app) + .post(`/libraries/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(204); + } + + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + + expect(stillOfflineAsset.isTrashed).toBe(true); + expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`); + expect(stillOfflineAsset.isOffline).toBe(true); + + { + const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true }); + expect(assets.count).toBe(1); + } + }); }); describe('POST /libraries/:id/validate', () => { diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index 14225ff063038..b00c3c0b6d30d 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -10,6 +10,7 @@ import { Permission, PersonCreateDto, SharedLinkCreateDto, + UpdateLibraryDto, UserAdminCreateDto, UserPreferencesUpdateDto, ValidateLibraryDto, @@ -35,6 +36,7 @@ import { updateAlbumUser, updateAssets, updateConfig, + updateLibrary, updateMyPreferences, upsertTags, validate, @@ -42,7 +44,7 @@ import { import { BrowserContext } from '@playwright/test'; import { exec, spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path, { dirname } from 'node:path'; import { setTimeout as setAsyncTimeout } from 'node:timers/promises'; @@ -392,6 +394,14 @@ export const utils = { rmSync(path); }, + renameImageFile: (oldPath: string, newPath: string) => { + if (!existsSync(oldPath)) { + return; + } + + renameSync(oldPath, newPath); + }, + removeDirectory: (path: string) => { if (!existsSync(path)) { return; @@ -444,6 +454,9 @@ export const utils = { createLibrary: (accessToken: string, dto: CreateLibraryDto) => createLibrary({ createLibraryDto: dto }, { headers: asBearerAuth(accessToken) }), + updateLibrary: (accessToken: string, id: string, dto: UpdateLibraryDto) => + updateLibrary({ id, updateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }), + validateLibrary: (accessToken: string, id: string, dto: ValidateLibraryDto) => validate({ id, validateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }), diff --git a/e2e/test-assets b/e2e/test-assets index 99544a200412d..c4a0575c3e89a 160000 --- a/e2e/test-assets +++ b/e2e/test-assets @@ -1 +1 @@ -Subproject commit 99544a200412d553103cc7b8f1a28f339c7cffd9 +Subproject commit c4a0575c3e89a755b951ae6d91e7307cd34c606f diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index b25e42ba0e9e0..35bb41aac068d 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -1,10 +1,11 @@ import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; +import { LibraryEntity } from 'src/entities/library.entity'; import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum'; import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; import { Paginated, PaginationOptions } from 'src/utils/pagination'; -import { FindOptionsOrder, FindOptionsRelations, FindOptionsSelect } from 'typeorm'; +import { FindOptionsOrder, FindOptionsRelations, FindOptionsSelect, UpdateResult } from 'typeorm'; export type AssetStats = Record; @@ -155,6 +156,7 @@ export const IAssetRepository = 'IAssetRepository'; export interface IAssetRepository { create(asset: AssetCreate): Promise; + createAll(assets: AssetCreate[]): Promise; getByIds( ids: string[], relations?: FindOptionsRelations, @@ -197,4 +199,6 @@ export interface IAssetRepository { getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise; upsertFile(file: UpsertFileOptions): Promise; upsertFiles(files: UpsertFileOptions[]): Promise; + updateOffline(library: LibraryEntity): Promise; + getNewPaths(libraryId: string, paths: string[]): Promise; } diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts index 7976f813022ff..424691f087e09 100644 --- a/server/src/interfaces/job.interface.ts +++ b/server/src/interfaces/job.interface.ts @@ -84,8 +84,8 @@ export enum JobName { // library management LIBRARY_QUEUE_SYNC_FILES = 'library-queue-sync-files', LIBRARY_QUEUE_SYNC_ASSETS = 'library-queue-sync-assets', - LIBRARY_SYNC_FILE = 'library-sync-file', - LIBRARY_SYNC_ASSET = 'library-sync-asset', + LIBRARY_SYNC_FILES = 'library-sync-files', + LIBRARY_SYNC_ASSETS = 'library-sync-assets', LIBRARY_DELETE = 'library-delete', LIBRARY_QUEUE_SYNC_ALL = 'library-queue-sync-all', LIBRARY_QUEUE_CLEANUP = 'library-queue-cleanup', @@ -135,7 +135,7 @@ export interface IDelayedJob extends IBaseJob { export interface IEntityJob extends IBaseJob { id: string; - source?: 'upload' | 'sidecar-write' | 'copy'; + source?: 'upload' | 'library-import' | 'sidecar-write' | 'copy'; notify?: boolean; } @@ -143,20 +143,26 @@ export interface IAssetDeleteJob extends IEntityJob { deleteOnDisk: boolean; } -export interface ILibraryFileJob extends IEntityJob { +export interface ILibraryFileJob { + libraryId: string; ownerId: string; - assetPath: string; + assetPaths: string[]; } -export interface ILibraryAssetJob extends IEntityJob { - importPaths: string[]; - exclusionPatterns: string[]; +export interface ILibraryBulkIdsJob { + libraryId: string; + assetIds: string[]; } -export interface IBulkEntityJob extends IBaseJob { +export interface IBulkEntityJob { ids: string[]; } +export interface ILibraryAssetsJob extends IBulkEntityJob { + importPaths: string[]; + exclusionPatterns: string[]; +} + export interface IDeleteFilesJob extends IBaseJob { files: Array; } @@ -284,10 +290,10 @@ export type JobItem = | { name: JobName.ASSET_DELETION_CHECK; data?: IBaseJob } // Library Management - | { name: JobName.LIBRARY_SYNC_FILE; data: ILibraryFileJob } + | { name: JobName.LIBRARY_SYNC_FILES; data: ILibraryFileJob } | { name: JobName.LIBRARY_QUEUE_SYNC_FILES; data: IEntityJob } | { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS; data: IEntityJob } - | { name: JobName.LIBRARY_SYNC_ASSET; data: ILibraryAssetJob } + | { name: JobName.LIBRARY_SYNC_ASSETS; data: ILibraryBulkIdsJob } | { name: JobName.LIBRARY_DELETE; data: IEntityJob } | { name: JobName.LIBRARY_QUEUE_SYNC_ALL; data?: IBaseJob } | { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob } diff --git a/server/src/interfaces/library.interface.ts b/server/src/interfaces/library.interface.ts index d8f1a1303116e..803cf1bc4ed31 100644 --- a/server/src/interfaces/library.interface.ts +++ b/server/src/interfaces/library.interface.ts @@ -1,8 +1,15 @@ +import { ADDED_IN_PREFIX } from 'src/constants'; import { LibraryStatsResponseDto } from 'src/dtos/library.dto'; import { LibraryEntity } from 'src/entities/library.entity'; export const ILibraryRepository = 'ILibraryRepository'; +export enum AssetSyncResult { + DO_NOTHING, + UPDATE, + OFFLINE, +} + export interface ILibraryRepository { getAll(withDeleted?: boolean): Promise; getAllDeleted(): Promise; diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 33d1e2457eb85..da619f0d356b2 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -5,6 +5,7 @@ import { AssetFileEntity } from 'src/entities/asset-files.entity'; import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; +import { LibraryEntity } from 'src/entities/library.entity'; import { AssetFileType, AssetOrder, AssetStatus, AssetType, PaginationMode } from 'src/enum'; import { AssetBuilderOptions, @@ -29,9 +30,11 @@ import { } from 'src/interfaces/asset.interface'; import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; import { searchAssetBuilder } from 'src/utils/database'; +import { globToSqlPattern } from 'src/utils/misc'; import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination'; import { Brackets, + DataSource, FindOptionsOrder, FindOptionsRelations, FindOptionsSelect, @@ -41,6 +44,7 @@ import { MoreThan, Not, Repository, + UpdateResult, } from 'typeorm'; const truncateMap: Record = { @@ -60,6 +64,7 @@ export class AssetRepository implements IAssetRepository { @InjectRepository(AssetFileEntity) private fileRepository: Repository, @InjectRepository(ExifEntity) private exifRepository: Repository, @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository, + private dataSource: DataSource, ) {} async upsertExif(exif: Partial): Promise { @@ -74,6 +79,10 @@ export class AssetRepository implements IAssetRepository { return this.repository.save(asset); } + createAll(assets: AssetCreate[]): Promise { + return this.repository.save(assets); + } + @GenerateSql({ params: [[DummyValue.UUID], { day: 1, month: 1 }] }) async getByDayOfYear(ownerIds: string[], { day, month }: MonthDay): Promise { const assets = await this.repository @@ -726,4 +735,48 @@ export class AssetRepository implements IAssetRepository { async upsertFiles(files: { assetId: string; type: AssetFileType; path: string }[]): Promise { await this.fileRepository.upsert(files, { conflictPaths: ['assetId', 'type'] }); } + + updateOffline(library: LibraryEntity): Promise { + const paths = library.importPaths.map((importPath) => `${importPath}%`).join('|'); + const exclusions = library.exclusionPatterns.map((pattern) => globToSqlPattern(pattern)).join('|'); + return this.repository + .createQueryBuilder() + .update() + .set({ + isOffline: true, + deletedAt: new Date(), + }) + .where({ isOffline: false }) + .andWhere({ libraryId: library.id }) + .andWhere( + new Brackets((qb) => { + qb.where('originalPath NOT SIMILAR TO :paths', { + paths, + }).orWhere('originalPath SIMILAR TO :exclusions', { + exclusions, + }); + }), + ) + .execute(); + } + + async getNewPaths(libraryId: string, paths: string[]): Promise { + const rawSql = ` + WITH unnested_paths AS ( + SELECT unnest($1::text[]) AS path + ) + SELECT unnested_paths.path AS path + FROM unnested_paths + WHERE not exists( + SELECT 1 + FROM assets + WHERE "originalPath" = unnested_paths.path AND + "libraryId" = $2 + ); + `; + + return this.repository + .query(rawSql, [paths, libraryId]) + .then((result) => result.map((row: { path: string }) => row.path)); + } } diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index 43d6662d659e5..6441634d43b06 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -10,7 +10,7 @@ import { ICronRepository } from 'src/interfaces/cron.interface'; import { IDatabaseRepository } from 'src/interfaces/database.interface'; import { IJobRepository, - ILibraryAssetJob, + ILibraryAssetsJob, ILibraryFileJob, JobName, JOBS_LIBRARY_PAGINATION_SIZE, @@ -179,7 +179,7 @@ describe(LibraryService.name, () => { expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.LIBRARY_SYNC_FILE, + name: JobName.LIBRARY_SYNC_FILES, data: { id: libraryStub.externalLibrary1.id, ownerId: libraryStub.externalLibrary1.owner.id, @@ -231,7 +231,7 @@ describe(LibraryService.name, () => { expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.LIBRARY_SYNC_ASSET, + name: JobName.LIBRARY_SYNC_ASSETS, data: { id: assetStub.external.id, importPaths: libraryStub.externalLibrary1.importPaths, @@ -250,22 +250,22 @@ describe(LibraryService.name, () => { describe('handleSyncAsset', () => { it('should skip missing assets', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/'], exclusionPatterns: [], }; assetMock.getById.mockResolvedValue(null); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SKIPPED); expect(assetMock.remove).not.toHaveBeenCalled(); }); it('should offline assets no longer on disk', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/'], exclusionPatterns: [], }; @@ -273,7 +273,7 @@ describe(LibraryService.name, () => { assetMock.getById.mockResolvedValue(assetStub.external); storageMock.stat.mockRejectedValue(new Error('ENOENT, no such file or directory')); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.external.id], { isOffline: true, @@ -282,15 +282,15 @@ describe(LibraryService.name, () => { }); it('should offline assets matching an exclusion pattern', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/'], exclusionPatterns: ['**/user1/**'], }; assetMock.getById.mockResolvedValue(assetStub.external); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.external.id], { isOffline: true, deletedAt: expect.any(Date), @@ -298,8 +298,8 @@ describe(LibraryService.name, () => { }); it('should set assets outside of import paths as offline', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/data/user2'], exclusionPatterns: [], }; @@ -307,7 +307,7 @@ describe(LibraryService.name, () => { assetMock.getById.mockResolvedValue(assetStub.external); storageMock.checkFileExists.mockResolvedValue(true); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.external.id], { isOffline: true, @@ -316,8 +316,8 @@ describe(LibraryService.name, () => { }); it('should do nothing with online assets', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/'], exclusionPatterns: [], }; @@ -325,14 +325,14 @@ describe(LibraryService.name, () => { assetMock.getById.mockResolvedValue(assetStub.external); storageMock.stat.mockResolvedValue({ mtime: assetStub.external.fileModifiedAt } as Stats); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); expect(assetMock.updateAll).not.toHaveBeenCalled(); }); it('should un-trash an asset previously marked as offline', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/'], exclusionPatterns: [], }; @@ -340,7 +340,7 @@ describe(LibraryService.name, () => { assetMock.getById.mockResolvedValue(assetStub.trashedOffline); storageMock.stat.mockResolvedValue({ mtime: assetStub.trashedOffline.fileModifiedAt } as Stats); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.trashedOffline.id], { deletedAt: null, @@ -353,8 +353,8 @@ describe(LibraryService.name, () => { }); it('should update file when mtime has changed', async () => { - const mockAssetJob: ILibraryAssetJob = { - id: assetStub.external.id, + const mockAssetJob: ILibraryAssetsJob = { + ids: [assetStub.external.id], importPaths: ['/'], exclusionPatterns: [], }; @@ -363,7 +363,7 @@ describe(LibraryService.name, () => { assetMock.getById.mockResolvedValue(assetStub.external); storageMock.stat.mockResolvedValue({ mtime: newMTime } as Stats); - await expect(sut.handleSyncAsset(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); + await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS); expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.external.id], { fileModifiedAt: newMTime, @@ -960,7 +960,7 @@ describe(LibraryService.name, () => { expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.LIBRARY_SYNC_FILE, + name: JobName.LIBRARY_SYNC_FILES, data: { id: libraryStub.externalLibraryWithImportPaths1.id, assetPath: '/foo/photo.jpg', @@ -969,7 +969,7 @@ describe(LibraryService.name, () => { }, ]); expect(jobMock.queueAll).toHaveBeenCalledWith([ - { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) }, + { name: JobName.LIBRARY_SYNC_ASSETS, data: expect.objectContaining({ ids: [assetStub.image.id] }) }, ]); }); @@ -985,7 +985,7 @@ describe(LibraryService.name, () => { expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.LIBRARY_SYNC_FILE, + name: JobName.LIBRARY_SYNC_FILES, data: { id: libraryStub.externalLibraryWithImportPaths1.id, assetPath: '/foo/photo.jpg', @@ -994,7 +994,7 @@ describe(LibraryService.name, () => { }, ]); expect(jobMock.queueAll).toHaveBeenCalledWith([ - { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) }, + { name: JobName.LIBRARY_SYNC_ASSETS, data: expect.objectContaining({ id: [assetStub.image.id] }) }, ]); }); @@ -1009,7 +1009,7 @@ describe(LibraryService.name, () => { await sut.watchAll(); expect(jobMock.queueAll).toHaveBeenCalledWith([ - { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) }, + { name: JobName.LIBRARY_SYNC_ASSETS, data: expect.objectContaining({ ids: [assetStub.image.id] }) }, ]); }); @@ -1166,9 +1166,9 @@ describe(LibraryService.name, () => { expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.LIBRARY_SYNC_ASSET, + name: JobName.LIBRARY_SYNC_ASSETS, data: { - id: assetStub.image1.id, + ids: [assetStub.image1.id], importPaths: libraryStub.externalLibrary1.importPaths, exclusionPatterns: libraryStub.externalLibrary1.exclusionPatterns, }, diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index c0d24fea9e19d..dc391d241d58e 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -16,10 +16,12 @@ import { } from 'src/dtos/library.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { LibraryEntity } from 'src/entities/library.entity'; -import { AssetType, ImmichWorker } from 'src/enum'; +import { AssetStatus, AssetType, ImmichWorker } from 'src/enum'; +import { AssetCreate } from 'src/interfaces/asset.interface'; import { DatabaseLock } from 'src/interfaces/database.interface'; import { ArgOf } from 'src/interfaces/event.interface'; import { JobName, JobOf, JOBS_LIBRARY_PAGINATION_SIZE, JobStatus, QueueName } from 'src/interfaces/job.interface'; +import { AssetSyncResult } from 'src/interfaces/library.interface'; import { BaseService } from 'src/services/base.service'; import { mimeTypes } from 'src/utils/mime-types'; import { handlePromiseError } from 'src/utils/misc'; @@ -98,6 +100,18 @@ export class LibraryService extends BaseService { let _resolve: () => void; const ready$ = new Promise((resolve) => (_resolve = resolve)); + const handler = async (event: string, path: string) => { + if (matcher(path)) { + this.logger.debug(`File ${event} event received for ${path} in library ${library.id}}`); + await this.jobRepository.queue({ + name: JobName.LIBRARY_SYNC_FILES, + data: { libraryId: library.id, ownerId: library.ownerId, assetPaths: [path] }, + }); + } else { + this.logger.verbose(`Ignoring file ${event} event for ${path} in library ${library.id}`); + } + }; + this.watchers[id] = this.storageRepository.watch( library.importPaths, { @@ -107,43 +121,13 @@ export class LibraryService extends BaseService { { onReady: () => _resolve(), onAdd: (path) => { - const handler = async () => { - this.logger.debug(`File add event received for ${path} in library ${library.id}}`); - if (matcher(path)) { - const asset = await this.assetRepository.getByLibraryIdAndOriginalPath(library.id, path); - if (asset) { - await this.syncAssets(library, [asset.id]); - } - if (matcher(path)) { - await this.syncFiles(library, [path]); - } - } - }; - return handlePromiseError(handler(), this.logger); + return handlePromiseError(handler('add', path), this.logger); }, onChange: (path) => { - const handler = async () => { - this.logger.debug(`Detected file change for ${path} in library ${library.id}`); - const asset = await this.assetRepository.getByLibraryIdAndOriginalPath(library.id, path); - if (asset) { - await this.syncAssets(library, [asset.id]); - } - if (matcher(path)) { - // Note: if the changed file was not previously imported, it will be imported now. - await this.syncFiles(library, [path]); - } - }; - return handlePromiseError(handler(), this.logger); + return handlePromiseError(handler('change', path), this.logger); }, onUnlink: (path) => { - const handler = async () => { - this.logger.debug(`Detected deleted file at ${path} in library ${library.id}`); - const asset = await this.assetRepository.getByLibraryIdAndOriginalPath(library.id, path); - if (asset) { - await this.syncAssets(library, [asset.id]); - } - }; - return handlePromiseError(handler(), this.logger); + return handlePromiseError(handler('delete', path), this.logger); }, onError: (error) => { this.logger.error(`Library watcher for library ${library.id} encountered error: ${error}`); @@ -228,26 +212,32 @@ export class LibraryService extends BaseService { return mapLibrary(library); } - private async syncFiles({ id, ownerId }: LibraryEntity, assetPaths: string[]) { - await this.jobRepository.queueAll( - assetPaths.map((assetPath) => ({ - name: JobName.LIBRARY_SYNC_FILE, - data: { - id, - assetPath, - ownerId, - }, - })), - ); - } + @OnJob({ name: JobName.LIBRARY_SYNC_FILES, queue: QueueName.LIBRARY }) + async handleSyncFiles(job: JobOf): Promise { + const library = await this.libraryRepository.get(job.libraryId); + if (!library) { + // We need to check if the library still exists as it could have been deleted after the scan was queued + this.logger.debug(`Library ${job.libraryId} not found, skipping file import`); + return JobStatus.FAILED; + } - private async syncAssets({ importPaths, exclusionPatterns }: LibraryEntity, assetIds: string[]) { - await this.jobRepository.queueAll( - assetIds.map((assetId) => ({ - name: JobName.LIBRARY_SYNC_ASSET, - data: { id: assetId, importPaths, exclusionPatterns }, - })), - ); + const assetImports = job.assetPaths.map((assetPath) => this.processEntity(assetPath, job.ownerId, job.libraryId)); + + const assetIds: string[] = []; + + // Due to a typeorm limitation we must batch the inserts + const batchSize = 2000; + for (let i = 0; i < assetImports.length; i += batchSize) { + const batch = assetImports.slice(i, i + batchSize); + const batchIds = await this.assetRepository.createAll(batch).then((assets) => assets.map((asset) => asset.id)); + assetIds.push(...batchIds); + } + + this.logger.log(`Imported ${assetIds.length} file(s) into library ${job.libraryId}`); + + await this.queuePostSyncJobs(assetIds); + + return JobStatus.SUCCESS; } private async validateImportPath(importPath: string): Promise { @@ -361,98 +351,62 @@ export class LibraryService extends BaseService { return JobStatus.SUCCESS; } - @OnJob({ name: JobName.LIBRARY_SYNC_FILE, queue: QueueName.LIBRARY }) - async handleSyncFile(job: JobOf): Promise { - // Only needs to handle new assets - const assetPath = path.normalize(job.assetPath); - - let asset = await this.assetRepository.getByLibraryIdAndOriginalPath(job.id, assetPath); - if (asset) { - return JobStatus.SKIPPED; - } - - let stat; - try { - stat = await this.storageRepository.stat(assetPath); - } catch (error: any) { - if (error.code === 'ENOENT') { - this.logger.error(`File not found: ${assetPath}`); - return JobStatus.SKIPPED; - } - this.logger.error(`Error reading file: ${assetPath}. Error: ${error}`); - return JobStatus.FAILED; - } + private processEntity(filePath: string, ownerId: string, libraryId: string): AssetCreate { + const assetPath = path.normalize(filePath); - this.logger.log(`Importing new library asset: ${assetPath}`); + const now = new Date(); - const library = await this.libraryRepository.get(job.id, true); - if (!library || library.deletedAt) { - this.logger.error('Cannot import asset into deleted library'); - return JobStatus.FAILED; - } - - // TODO: device asset id is deprecated, remove it - const deviceAssetId = `${basename(assetPath)}`.replaceAll(/\s+/g, ''); - - const pathHash = this.cryptoRepository.hashSha1(`path:${assetPath}`); - - // TODO: doesn't xmp replace the file extension? Will need investigation - let sidecarPath: string | null = null; - if (await this.storageRepository.checkFileExists(`${assetPath}.xmp`, R_OK)) { - sidecarPath = `${assetPath}.xmp`; - } - - const assetType = mimeTypes.isVideo(assetPath) ? AssetType.VIDEO : AssetType.IMAGE; - - const mtime = stat.mtime; - - asset = await this.assetRepository.create({ - ownerId: job.ownerId, - libraryId: job.id, - checksum: pathHash, + return { + ownerId: ownerId, + libraryId: libraryId, + checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`), originalPath: assetPath, - deviceAssetId, + + // TODO: device asset id is deprecated, remove it + deviceAssetId: `${basename(assetPath)}`.replaceAll(/\s+/g, ''), deviceId: 'Library Import', - fileCreatedAt: mtime, - fileModifiedAt: mtime, - localDateTime: mtime, - type: assetType, + fileCreatedAt: now, + fileModifiedAt: now, + localDateTime: now, + type: mimeTypes.isVideo(assetPath) ? AssetType.VIDEO : AssetType.IMAGE, originalFileName: parse(assetPath).base, - - sidecarPath, isExternal: true, - }); - - await this.queuePostSyncJobs(asset); - - return JobStatus.SUCCESS; + }; } - async queuePostSyncJobs(asset: AssetEntity) { - this.logger.debug(`Queueing metadata extraction for: ${asset.originalPath}`); - - await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id, source: 'upload' } }); + async queuePostSyncJobs(assetIds: string[]) { + await this.jobRepository.queueAll( + assetIds.map((assetId) => ({ + name: JobName.METADATA_EXTRACTION, + data: { id: assetId, source: 'library-import' }, + })), + ); } async queueScan(id: string) { await this.findOrFail(id); - await this.jobRepository.queue({ - name: JobName.LIBRARY_QUEUE_SYNC_FILES, - data: { - id, + this.logger.log(`Starting to scan library ${id}`); + + await this.jobRepository.queueAll([ + { + name: JobName.LIBRARY_QUEUE_SYNC_FILES, + data: { + id, + }, }, - }); - await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } }); + { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } }, + ]); } @OnJob({ name: JobName.LIBRARY_QUEUE_SYNC_ALL, queue: QueueName.LIBRARY }) async handleQueueSyncAll(): Promise { - this.logger.debug(`Refreshing all external libraries`); + this.logger.log(`Initiating scan of all external libraries`); await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_CLEANUP, data: {} }); const libraries = await this.libraryRepository.getAll(true); + await this.jobRepository.queueAll( libraries.map((library) => ({ name: JobName.LIBRARY_QUEUE_SYNC_FILES, @@ -469,63 +423,116 @@ export class LibraryService extends BaseService { }, })), ); + return JobStatus.SUCCESS; } - @OnJob({ name: JobName.LIBRARY_SYNC_ASSET, queue: QueueName.LIBRARY }) - async handleSyncAsset(job: JobOf): Promise { - const asset = await this.assetRepository.getById(job.id); - if (!asset) { - return JobStatus.SKIPPED; + @OnJob({ name: JobName.LIBRARY_SYNC_ASSETS, queue: QueueName.LIBRARY }) + async handleSyncAssets(job: JobOf): Promise { + const assets = await this.assetRepository.getByIds(job.assetIds); + + const assetIdsToOffline: string[] = []; + const assetIdsToUpdate: string[] = []; + + for (const asset of assets) { + const action = await this.handleSyncAsset(asset); + switch (action) { + case AssetSyncResult.OFFLINE: + assetIdsToOffline.push(asset.id); + break; + case AssetSyncResult.UPDATE: + assetIdsToUpdate.push(asset.id); + break; + } } - const markOffline = async (explanation: string) => { - if (!asset.isOffline) { - this.logger.debug(`${explanation}, removing: ${asset.originalPath}`); - await this.assetRepository.updateAll([asset.id], { isOffline: true, deletedAt: new Date() }); - } - }; + if (assetIdsToOffline.length) { + await this.assetRepository.updateAll(assetIdsToOffline, { isOffline: true, deletedAt: new Date() }); + this.logger.log( + `Originals are missing for ${assetIdsToOffline.length} asset(s) in library ${job.libraryId}, marked offline`, + ); + } + + if (assetIdsToUpdate.length) { + //TODO: When we have asset status, we need to leave deletedAt as is when status is trashed + await this.assetRepository.updateAll(assetIdsToUpdate, { + isOffline: false, + deletedAt: null, + }); + + this.logger.log( + `Found ${assetIdsToOffline.length} asset(s) with modified files for library ${job.libraryId}, queuing refresh...`, + ); + + await this.queuePostSyncJobs(assetIdsToUpdate); + } + + const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length; + if (remainingCount > 0) { + this.logger.log(`${remainingCount} asset(s) are unchanged in library ${job.libraryId}, no action required`); + } - const isInPath = job.importPaths.find((path) => asset.originalPath.startsWith(path)); - if (!isInPath) { - await markOffline('Asset is no longer in an import path'); - return JobStatus.SUCCESS; + return JobStatus.SUCCESS; + } + + private async checkOfflineAsset(asset: AssetEntity) { + if (!asset.libraryId) { + return false; } - const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern)); + const library = await this.libraryRepository.get(asset.libraryId); + if (!library) { + return false; + } + + const isInImportPath = library.importPaths.find((path) => asset.originalPath.startsWith(path)); + if (!isInImportPath) { + return false; + } + + const isExcluded = library.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern)); if (isExcluded) { - await markOffline('Asset is covered by an exclusion pattern'); - return JobStatus.SUCCESS; + return false; + } + + return true; + } + + private async handleSyncAsset(asset: AssetEntity): Promise { + if (!asset) { + return AssetSyncResult.DO_NOTHING; } let stat; try { stat = await this.storageRepository.stat(asset.originalPath); } catch { - await markOffline('Asset is no longer on disk or is inaccessible because of permissions'); - return JobStatus.SUCCESS; + if (asset.isOffline) { + return AssetSyncResult.DO_NOTHING; + } + + this.logger.debug( + `Asset is no longer on disk or is inaccessible because of permissions, moving to trash: ${asset.originalPath}`, + ); + return AssetSyncResult.OFFLINE; } const mtime = stat.mtime; const isAssetModified = mtime.toISOString() !== asset.fileModifiedAt.toISOString(); + let shouldAssetGoOnline = false; - if (asset.isOffline || isAssetModified) { - this.logger.debug(`Asset was offline or modified, updating asset record ${asset.originalPath}`); - //TODO: When we have asset status, we need to leave deletedAt as is when status is trashed - await this.assetRepository.updateAll([asset.id], { - isOffline: false, - deletedAt: null, - fileCreatedAt: mtime, - fileModifiedAt: mtime, - originalFileName: parse(asset.originalPath).base, - }); + if (asset.isOffline && asset.status != AssetStatus.DELETED) { + // Only perform the expensive check if the asset is offline + shouldAssetGoOnline = await this.checkOfflineAsset(asset); } - if (isAssetModified) { - this.logger.debug(`Asset was modified, queuing metadata extraction for: ${asset.originalPath}`); - await this.queuePostSyncJobs(asset); + if (shouldAssetGoOnline || isAssetModified) { + this.logger.debug(`Asset was offline or modified, updating asset record ${asset.originalPath}`); + + return AssetSyncResult.UPDATE; } - return JobStatus.SUCCESS; + + return AssetSyncResult.DO_NOTHING; } @OnJob({ name: JobName.LIBRARY_QUEUE_SYNC_FILES, queue: QueueName.LIBRARY }) @@ -536,7 +543,7 @@ export class LibraryService extends BaseService { return JobStatus.SKIPPED; } - this.logger.log(`Refreshing library ${library.id} for new assets`); + this.logger.debug(`Validating import paths for library ${library.id}...`); const validImportPaths: string[] = []; @@ -551,28 +558,55 @@ export class LibraryService extends BaseService { if (validImportPaths.length === 0) { this.logger.warn(`No valid import paths found for library ${library.id}`); + + return JobStatus.SKIPPED; } - const assetsOnDisk = this.storageRepository.walk({ + const pathsOnDisk = this.storageRepository.walk({ pathsToCrawl: validImportPaths, includeHidden: false, exclusionPatterns: library.exclusionPatterns, take: JOBS_LIBRARY_PAGINATION_SIZE, }); - let count = 0; + let importCount = 0; + let crawlCount = 0; + + this.logger.log(`Starting crawl of ${validImportPaths.length} path(s) for library ${library.id}...`); - for await (const assetBatch of assetsOnDisk) { - count += assetBatch.length; - this.logger.debug(`Discovered ${count} asset(s) on disk for library ${library.id}...`); - await this.syncFiles(library, assetBatch); - this.logger.verbose(`Queued scan of ${assetBatch.length} crawled asset(s) in library ${library.id}...`); + for await (const pathBatch of pathsOnDisk) { + crawlCount += pathBatch.length; + this.logger.debug( + `Crawled ${pathBatch.length} file(s) for library ${library.id}, in total ${crawlCount} file(s) crawled so far`, + ); + const newPaths = await this.assetRepository.getNewPaths(library.id, pathBatch); + if (newPaths.length > 0) { + importCount += newPaths.length; + + await this.jobRepository.queue({ + name: JobName.LIBRARY_SYNC_FILES, + data: { libraryId: library.id, ownerId: library.ownerId, assetPaths: newPaths }, + }); + this.logger.log( + `Crawled ${crawlCount} file(s) so far: ${newPaths.length} of current batch queued for import for ${library.id}...`, + ); + } else { + this.logger.log( + `Crawled ${crawlCount} file(s) so far: ${pathBatch.length} of current batch already in library ${library.id}...`, + ); + } } - if (count > 0) { - this.logger.debug(`Finished queueing scan of ${count} assets on disk for library ${library.id}`); - } else if (validImportPaths.length > 0) { - this.logger.debug(`No non-excluded assets found in any import path for library ${library.id}`); + if (crawlCount === 0) { + this.logger.log(`No files found on disk for library ${library.id}`); + } else if (importCount > 0 && importCount === crawlCount) { + this.logger.log(`Finished crawling and queueing ${crawlCount} file(s) for import for library ${library.id}`); + } else if (importCount > 0) { + this.logger.log( + `Finished crawling ${crawlCount} file(s) of which ${importCount} file(s) are queued for import for library ${library.id}`, + ); + } else { + this.logger.log(`All ${crawlCount} file(s) on disk are already in ${library.id}`); } await this.libraryRepository.update({ id: job.id, refreshedAt: new Date() }); @@ -587,27 +621,58 @@ export class LibraryService extends BaseService { return JobStatus.SKIPPED; } - this.logger.log(`Scanning library ${library.id} for removed assets`); + const assetCount = (await this.getStatistics(library.id)).total; + + this.logger.log( + `Scanning library ${library.id} for assets outside of import paths and/or matching an exclusion pattern...`, + ); + const offlineResult = await this.assetRepository.updateOffline(library); - const onlineAssets = usePagination(JOBS_LIBRARY_PAGINATION_SIZE, (pagination) => + const affectedAssetCount = offlineResult.affected; + if (affectedAssetCount === undefined) { + this.logger.error(`Unknown error occurred when updating offline status in ${library.id}`); + return JobStatus.FAILED; + } + + if (affectedAssetCount === assetCount) { + this.logger.log( + `All ${assetCount} asset(s) in ${library.id} are outside of import paths and/or match an exclusion pattern, marked as offline`, + ); + } else if (affectedAssetCount !== assetCount && affectedAssetCount > 0) { + this.logger.log( + `${offlineResult.affected} asset(s) out of ${assetCount} were marked offline due to import paths and/or exclusion patterns for library ${library.id}`, + ); + } else { + this.logger.log( + `All ${assetCount} asset(s) in library ${library.id} were in an import path and none matched an exclusion pattern`, + ); + } + + this.logger.log(`Scanning library ${library.id} for assets missing from disk...`); + + const existingAssets = usePagination(JOBS_LIBRARY_PAGINATION_SIZE, (pagination) => this.assetRepository.getAll(pagination, { libraryId: job.id, withDeleted: true }), ); - let assetCount = 0; - for await (const assets of onlineAssets) { - assetCount += assets.length; - this.logger.debug(`Discovered ${assetCount} asset(s) in library ${library.id}...`); - await this.jobRepository.queueAll( - assets.map((asset) => ({ - name: JobName.LIBRARY_SYNC_ASSET, - data: { id: asset.id, importPaths: library.importPaths, exclusionPatterns: library.exclusionPatterns }, - })), + let currentAssetCount = 0; + for await (const assets of existingAssets) { + currentAssetCount += assets.length; + + await this.jobRepository.queue({ + name: JobName.LIBRARY_SYNC_ASSETS, + data: { + libraryId: library.id, + assetIds: assets.map((asset) => asset.id), + }, + }); + + this.logger.log( + `Queued check of ${assets.length} existing asset(s) in library ${library.id}, ${currentAssetCount} of ${assetCount} queued in total`, ); - this.logger.debug(`Queued check of ${assets.length} asset(s) in library ${library.id}...`); } - if (assetCount) { - this.logger.log(`Finished queueing check of ${assetCount} assets for library ${library.id}`); + if (currentAssetCount) { + this.logger.log(`Finished queuing ${currentAssetCount} file checks for library ${library.id}`); } return JobStatus.SUCCESS; diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 79a7d519d601e..14dae28da0ccc 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -148,13 +148,17 @@ export class MetadataService extends BaseService { } @OnJob({ name: JobName.METADATA_EXTRACTION, queue: QueueName.METADATA_EXTRACTION }) - async handleMetadataExtraction({ id }: JobOf): Promise { + async handleMetadataExtraction({ id, source }: JobOf): Promise { const { metadata, reverseGeocoding } = await this.getConfig({ withCache: true }); const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } }); if (!asset) { return JobStatus.FAILED; } + if (source === 'library-import') { + await this.processSidecar(id, false); + } + const stats = await this.storageRepository.stat(asset.originalPath); const exifTags = await this.getExifTags(asset); diff --git a/server/src/utils/misc.spec.ts b/server/src/utils/misc.spec.ts index 53be77dc21a58..87ab6d4399bbf 100644 --- a/server/src/utils/misc.spec.ts +++ b/server/src/utils/misc.spec.ts @@ -1,4 +1,4 @@ -import { getKeysDeep, unsetDeep } from 'src/utils/misc'; +import { getKeysDeep, globToSqlPattern, unsetDeep } from 'src/utils/misc'; import { describe, expect, it } from 'vitest'; describe('getKeysDeep', () => { @@ -51,3 +51,18 @@ describe('unsetDeep', () => { expect(unsetDeep({ foo: 'bar', nested: { enabled: true } }, 'nested.enabled')).toEqual({ foo: 'bar' }); }); }); + +describe('globToSqlPattern', () => { + const testCases = [ + ['**/Raw/**', '%/Raw/%'], + ['**/abc/*.tif', '%/abc/%.tif'], + ['**/*.tif', '%/%.tif'], + ['**/*.jp?', '%/%.jp_'], + ['**/@eaDir/**', '%/@eaDir/%'], + ['**/._*', `%/.\\_%`], + ]; + + it.each(testCases)('should convert %s to %s', (input, expected) => { + expect(globToSqlPattern(input)).toEqual(expected); + }); +}); diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 6a64923a3bf7b..3543cf20b02f5 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -10,6 +10,7 @@ import { ReferenceObject, SchemaObject } from '@nestjs/swagger/dist/interfaces/o import _ from 'lodash'; import { writeFileSync } from 'node:fs'; import path from 'node:path'; +import picomatch from 'picomatch'; import { SystemConfig } from 'src/config'; import { CLIP_MODEL_INFO, serverVersion } from 'src/constants'; import { ImmichCookie, ImmichHeader, MetadataKey } from 'src/enum'; @@ -264,3 +265,55 @@ export const useSwagger = (app: INestApplication, { write }: { write: boolean }) writeFileSync(outputPath, JSON.stringify(patchOpenAPI(specification), null, 2), { encoding: 'utf8' }); } }; + +const convertTokenToSqlPattern = (token: any): string => { + if (typeof token === 'string') { + return token; + } + + switch (token.type) { + case 'slash': { + return '/'; + } + case 'text': { + return token.value; + } + case 'globstar': + case 'star': { + return '%'; + } + case 'underscore': { + return String.raw`\_`; + } + case 'qmark': { + return '_'; + } + case 'dot': { + return '.'; + } + case 'bracket': { + return `[${token.value}]`; + } + case 'negate': { + return `[^${token.value}]`; + } + case 'brace': { + const options = token.value.split(','); + return `(${options.join('|')})`; + } + default: { + return ''; + } + } +}; + +export const globToSqlPattern = (glob: string) => { + const tokens = picomatch.parse(glob).tokens; + + let result = ''; + for (const token of tokens) { + result += convertTokenToSqlPattern(token); + } + + return result; +}; diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 928a7956c5f0c..07020c48c2c7e 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -38,5 +38,7 @@ export const newAssetRepositoryMock = (): Mocked => { getDuplicates: vitest.fn(), upsertFile: vitest.fn(), upsertFiles: vitest.fn(), + updateOffline: vitest.fn(), + getNewPaths: vitest.fn(), }; };