Skip to content

Commit

Permalink
wip move back to typeorm brackets
Browse files Browse the repository at this point in the history
  • Loading branch information
@etnoy
etnoy committed Dec 10, 2024
1 parent 02c5765 commit 6d69307
Show file tree
Hide file tree
Showing 5 changed files with 278 additions and 59 deletions.
197 changes: 193 additions & 4 deletions e2e/src/api/specs/library.e2e-spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk';
import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary, updateLibrary } from '@immich/sdk';

Check failure on line 1 in e2e/src/api/specs/library.e2e-spec.ts

View workflow job for this annotation

GitHub Actions / End-to-End Lint 

'updateLibrary' is defined but never used
import { cpSync, existsSync } from 'node:fs';
import { Socket } from 'socket.io-client';
import { userDto, uuidDto } from 'src/fixtures';
Expand Down Expand Up @@ -421,7 +421,7 @@ describe('/libraries', () => {
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send({ refreshModifiedFiles: true });
.send();
expect(status).toBe(204);

await utils.waitForQueueFinish(admin.accessToken, 'library');
Expand Down Expand Up @@ -453,7 +453,7 @@ describe('/libraries', () => {
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send({ refreshModifiedFiles: true });
.send();
expect(status).toBe(204);

await utils.waitForQueueFinish(admin.accessToken, 'library');
Expand Down Expand Up @@ -577,7 +577,7 @@ describe('/libraries', () => {
]);
});

it('should not trash an online asset', async () => {
it('should not set an asset offline if its file exists, is in an import path, and not covered by an exclusion pattern', async () => {
const library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,
importPaths: [`${testAssetDirInternal}/temp`],
Expand All @@ -601,6 +601,195 @@ describe('/libraries', () => {

expect(assets).toEqual(assetsBefore);
});

it('should set an offline asset to online if its file exists, is in an import path, and not covered by an exclusion pattern', async () => {
utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);

const library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,
importPaths: [`${testAssetDirInternal}/temp/offline`],
});

await scan(admin.accessToken, library.id);
await utils.waitForQueueFinish(admin.accessToken, 'library');

const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });

utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);

{
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
}

await utils.waitForQueueFinish(admin.accessToken, 'library');

const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
expect(offlineAsset.isTrashed).toBe(true);
expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
expect(offlineAsset.isOffline).toBe(true);

{
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
expect(assets.count).toBe(1);
}

utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);

{
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
}

await utils.waitForQueueFinish(admin.accessToken, 'library');

const backOnlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);

expect(backOnlineAsset.isTrashed).toBe(false);
expect(backOnlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
expect(backOnlineAsset.isOffline).toBe(false);

{
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
expect(assets.count).toBe(1);
}
});

it('should not set an offline asset to online if its file exists, is not covered by an exclusion pattern, but is outside of all import paths', async () => {
utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);

const library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,
importPaths: [`${testAssetDirInternal}/temp/offline`],
});

await scan(admin.accessToken, library.id);
await utils.waitForQueueFinish(admin.accessToken, 'library');

const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });

utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);

{
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
}

await utils.waitForQueueFinish(admin.accessToken, 'library');

{
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
expect(assets.count).toBe(1);
}

const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);

expect(offlineAsset.isTrashed).toBe(true);
expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
expect(offlineAsset.isOffline).toBe(true);

utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);

utils.createDirectory(`${testAssetDir}/temp/another-path/`);

await utils.updateLibrary(admin.accessToken, library.id, {
importPaths: [`${testAssetDirInternal}/temp/another-path`],
});

{
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
}

await utils.waitForQueueFinish(admin.accessToken, 'library');

const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);

expect(stillOfflineAsset.isTrashed).toBe(true);
expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
expect(stillOfflineAsset.isOffline).toBe(true);

{
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
expect(assets.count).toBe(1);
}

utils.removeDirectory(`${testAssetDir}/temp/another-path/`);
});

it('should not set an offline asset to online if its file exists, is in an import path, but is covered by an exclusion pattern', async () => {
utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);

const library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,
importPaths: [`${testAssetDirInternal}/temp/offline`],
});

await scan(admin.accessToken, library.id);
await utils.waitForQueueFinish(admin.accessToken, 'library');

const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });

utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);

{
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
}

await utils.waitForQueueFinish(admin.accessToken, 'library');

{
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
expect(assets.count).toBe(1);
}

const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);

expect(offlineAsset.isTrashed).toBe(true);
expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
expect(offlineAsset.isOffline).toBe(true);

utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);

await utils.updateLibrary(admin.accessToken, library.id, { exclusionPatterns: ['**/offline/**'] });

{
const { status } = await request(app)
.post(`/libraries/${library.id}/scan`)
.set('Authorization', `Bearer ${admin.accessToken}`)
.send();
expect(status).toBe(204);
}

await utils.waitForQueueFinish(admin.accessToken, 'library');

const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);

expect(stillOfflineAsset.isTrashed).toBe(true);
expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
expect(stillOfflineAsset.isOffline).toBe(true);

{
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
expect(assets.count).toBe(1);
}
});
});

describe('POST /libraries/:id/validate', () => {
Expand Down
15 changes: 14 additions & 1 deletion e2e/src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
Permission,
PersonCreateDto,
SharedLinkCreateDto,
UpdateLibraryDto,
UserAdminCreateDto,
UserPreferencesUpdateDto,
ValidateLibraryDto,
Expand All @@ -35,14 +36,15 @@ import {
updateAlbumUser,
updateAssets,
updateConfig,
updateLibrary,
updateMyPreferences,
upsertTags,
validate,
} from '@immich/sdk';
import { BrowserContext } from '@playwright/test';
import { exec, spawn } from 'node:child_process';
import { createHash } from 'node:crypto';
import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path, { dirname } from 'node:path';
import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
Expand Down Expand Up @@ -392,6 +394,14 @@ export const utils = {
rmSync(path);
},

renameImageFile: (oldPath: string, newPath: string) => {
if (!existsSync(oldPath)) {
return;
}

renameSync(oldPath, newPath);
},

removeDirectory: (path: string) => {
if (!existsSync(path)) {
return;
Expand Down Expand Up @@ -444,6 +454,9 @@ export const utils = {
createLibrary: (accessToken: string, dto: CreateLibraryDto) =>
createLibrary({ createLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

updateLibrary: (accessToken: string, id: string, dto: UpdateLibraryDto) =>
updateLibrary({ id, updateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

validateLibrary: (accessToken: string, id: string, dto: ValidateLibraryDto) =>
validate({ id, validateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

Expand Down
2 changes: 1 addition & 1 deletion server/src/interfaces/asset.interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,5 +193,5 @@ export interface IAssetRepository {
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
upsertFile(file: UpsertFileOptions): Promise<void>;
upsertFiles(files: UpsertFileOptions[]): Promise<void>;
updateOffline(pagination: PaginationOptions, library: LibraryEntity): Paginated<AssetEntity>;
updateOffline(importPaths: string[], exclusionPatterns: string[]): Promise<UpdateResult>;
}
55 changes: 21 additions & 34 deletions server/src/repositories/asset.repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -716,39 +716,26 @@ export class AssetRepository implements IAssetRepository {
await this.fileRepository.upsert(files, { conflictPaths: ['assetId', 'type'] });
}

updateOffline(pagination: PaginationOptions, library: LibraryEntity): Paginated<AssetEntity> {
return this.dataSource.manager.transaction(async (transactionalEntityManager) =>
transactionalEntityManager.query(
`
WITH updated_rows AS (
UPDATE assets
SET "isOffline" = $1, "deletedAt" = $2
WHERE "isOffline" = $3
AND (
"originalPath" NOT SIMILAR TO $4
OR "originalPath" SIMILAR TO $5
)
RETURNING id
)
SELECT *
FROM assets
WHERE id NOT IN (SELECT id FROM updated_rows)
AND "libraryId" = $6
AND ($7 OR "deletedAt" IS NULL)
LIMIT $8 OFFSET $9;
`,
[
true, // $1 - is_offline = true
new Date(), // $2 - deleted_at = current timestamp
false, // $3 - is_offline = false
library.importPaths.map((importPath) => `${importPath}%`).join('|'), // $4 - importPartMatcher pattern
library.exclusionPatterns.map(globToSqlPattern).join('|'), // $5 - exclusionPatternMatcher pattern
library.id, // $6 - libraryId matches job.id
true, // $7 - withDeleted flag
pagination.take, // $8 - LIMIT
pagination.skip, // $9 - OFFSET
],
),
);
updateOffline(importPaths: string[], exclusionPatterns: string[]): Promise<UpdateResult> {
const paths = importPaths.map((importPath) => `${importPath}%`).join('|');
const exclusions = exclusionPatterns.map(globToSqlPattern).join('|');

Check failure on line 721 in server/src/repositories/asset.repository.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server 

Do not pass function `globToSqlPattern` directly to `.map(…)`
return this.repository
.createQueryBuilder()
.update()
.set({
isOffline: true,
deletedAt: new Date(),
})
.where({ isOffline: false })
.andWhere(
new Brackets((qb) => {
qb.where('originalPath NOT SIMILAR TO :paths', {
paths,
}).orWhere('originalPath SIMILAR TO :exclusions', {
exclusions,
});
}),
)
.execute();
}
}
Loading

0 comments on commit 6d69307

Please sign in to comment.