Compare commits

..

7 Commits

Author SHA1 Message Date
Jonathan Jogenfors
fa9baad116 bump chokidar and fix Jasons comments 2026-03-25 22:35:53 +01:00
Jonathan Jogenfors
3e43da5d8d Merge branch 'main' of https://github.com/immich-app/immich into fix/library-watcher-excludes 2026-03-25 22:25:37 +01:00
Mees Frensel
b9b5dba037 fix(web): crop square ratio i18n (#27257) 2026-03-25 14:05:43 -05:00
renovate[bot]
8bfa75087c chore(deps): update base-image to v202603251709 (major) (#27273)
chore(deps): update base-image to v202603251709

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
2026-03-25 14:04:26 -05:00
bo0tzz
95280edd6c fix: let renovate update base images (#27272) 2026-03-25 18:00:40 +00:00
Jonathan Jogenfors
292fbef180 move to medium tests 2026-03-24 14:49:07 +01:00
Jonathan Jogenfors
05b07ca233 feat: add e2e tests for library watcher 2026-03-24 13:17:23 +01:00
14 changed files with 395 additions and 29 deletions

View File

@@ -866,6 +866,7 @@
"crop_aspect_ratio_fixed": "Fixed",
"crop_aspect_ratio_free": "Free",
"crop_aspect_ratio_original": "Original",
"crop_aspect_ratio_square": "Square",
"curated_object_page_title": "Things",
"current_device": "Current device",
"current_pin_code": "Current PIN code",

52
pnpm-lock.yaml generated
View File

@@ -431,8 +431,8 @@ importers:
specifier: ^5.51.0
version: 5.70.4
chokidar:
specifier: ^4.0.3
version: 4.0.3
specifier: ^5.0.0
version: 5.0.0
class-transformer:
specifier: ^0.5.1
version: 0.5.1
@@ -592,7 +592,7 @@ importers:
version: 11.0.16(@swc/core@1.15.18(@swc/helpers@0.5.17))(@types/node@24.12.0)(esbuild@0.27.3)
'@nestjs/schematics':
specifier: ^11.0.0
version: 11.0.9(chokidar@4.0.3)(typescript@5.9.3)
version: 11.0.9(chokidar@5.0.0)(typescript@5.9.3)
'@nestjs/testing':
specifier: ^11.0.4
version: 11.1.16(@nestjs/common@11.1.16(class-transformer@0.5.1)(class-validator@0.15.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16)(@nestjs/platform-express@11.1.16)
@@ -6050,6 +6050,10 @@ packages:
resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==}
engines: {node: '>= 14.16.0'}
chokidar@5.0.0:
resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==}
engines: {node: '>= 20.19.0'}
chownr@1.1.4:
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
@@ -10540,6 +10544,10 @@ packages:
resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==}
engines: {node: '>= 14.18.0'}
readdirp@5.0.0:
resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==}
engines: {node: '>= 20.19.0'}
recma-build-jsx@1.0.0:
resolution: {integrity: sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==}
@@ -12602,6 +12610,17 @@ snapshots:
optionalDependencies:
chokidar: 4.0.3
'@angular-devkit/core@19.2.17(chokidar@5.0.0)':
dependencies:
ajv: 8.17.1
ajv-formats: 3.0.1(ajv@8.17.1)
jsonc-parser: 3.3.1
picomatch: 4.0.2
rxjs: 7.8.1
source-map: 0.7.4
optionalDependencies:
chokidar: 5.0.0
'@angular-devkit/core@19.2.19(chokidar@4.0.3)':
dependencies:
ajv: 8.17.1
@@ -12635,6 +12654,16 @@ snapshots:
transitivePeerDependencies:
- chokidar
'@angular-devkit/schematics@19.2.17(chokidar@5.0.0)':
dependencies:
'@angular-devkit/core': 19.2.17(chokidar@5.0.0)
jsonc-parser: 3.3.1
magic-string: 0.30.17
ora: 5.4.1
rxjs: 7.8.1
transitivePeerDependencies:
- chokidar
'@angular-devkit/schematics@19.2.19(chokidar@4.0.3)':
dependencies:
'@angular-devkit/core': 19.2.19(chokidar@4.0.3)
@@ -15720,6 +15749,17 @@ snapshots:
transitivePeerDependencies:
- chokidar
'@nestjs/schematics@11.0.9(chokidar@5.0.0)(typescript@5.9.3)':
dependencies:
'@angular-devkit/core': 19.2.17(chokidar@5.0.0)
'@angular-devkit/schematics': 19.2.17(chokidar@5.0.0)
comment-json: 4.4.1
jsonc-parser: 3.3.1
pluralize: 8.0.0
typescript: 5.9.3
transitivePeerDependencies:
- chokidar
'@nestjs/swagger@11.2.6(@nestjs/common@11.1.16(class-transformer@0.5.1)(class-validator@0.15.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.16)(class-transformer@0.5.1)(class-validator@0.15.1)(reflect-metadata@0.2.2)':
dependencies:
'@microsoft/tsdoc': 0.16.0
@@ -18441,6 +18481,10 @@ snapshots:
dependencies:
readdirp: 4.1.2
chokidar@5.0.0:
dependencies:
readdirp: 5.0.0
chownr@1.1.4: {}
chownr@2.0.0: {}
@@ -23672,6 +23716,8 @@ snapshots:
readdirp@4.1.2: {}
readdirp@5.0.0: {}
recma-build-jsx@1.0.0:
dependencies:
'@types/estree': 1.0.8

View File

@@ -27,6 +27,10 @@
"matchUpdateTypes": ["major"],
"enabled": false
},
{
"matchPackageNames": ["ghcr.io/immich-app/base-server-*"],
"maxMajorIncrement": 0
},
{
"matchPackageNames": ["ruby"],
"groupName": "ruby",

View File

@@ -1,4 +1,4 @@
FROM ghcr.io/immich-app/base-server-dev:202603031112@sha256:837536db5fd9e432f0f474ef9b61712fe3b3815821c3e4edf5e5b0b1f1ed30ad AS builder
FROM ghcr.io/immich-app/base-server-dev:202603251709@sha256:2bf3053c732fcb87ec90c3c614632ac44847423468ccc57fd935bff771828d9d AS builder
ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
CI=1 \
COREPACK_HOME=/tmp \
@@ -71,7 +71,7 @@ RUN --mount=type=cache,id=pnpm-plugins,target=/buildcache/pnpm-store \
--mount=type=cache,id=mise-tools-${TARGETPLATFORM},target=/buildcache/mise \
cd plugins && mise run build
FROM ghcr.io/immich-app/base-server-prod:202603031112@sha256:bb8c8645ee61977140121e56ba09db7ae656a7506f9a6af1be8461b4d81fdf03
FROM ghcr.io/immich-app/base-server-prod:202603251709@sha256:17de30977ff87aa06758a56ad7f10d6b5c97bf9dab76e4ec4177a2a8d1b2b5f3
WORKDIR /usr/src/app
ENV NODE_ENV=production \

View File

@@ -1,5 +1,5 @@
# dev build
FROM ghcr.io/immich-app/base-server-dev:202603031112@sha256:837536db5fd9e432f0f474ef9b61712fe3b3815821c3e4edf5e5b0b1f1ed30ad AS dev
FROM ghcr.io/immich-app/base-server-dev:202603251709@sha256:2bf3053c732fcb87ec90c3c614632ac44847423468ccc57fd935bff771828d9d AS dev
ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
CI=1 \

View File

@@ -64,7 +64,7 @@
"bcrypt": "^6.0.0",
"body-parser": "^2.2.0",
"bullmq": "^5.51.0",
"chokidar": "^4.0.3",
"chokidar": "^5.0.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.15.0",
"compression": "^1.8.0",

View File

@@ -582,6 +582,7 @@ where
"asset_face"."updateId" < $1
and "asset_face"."updateId" > $2
and "asset"."ownerId" = $3
and "asset_face"."isVisible" = $4
order by
"asset_face"."updateId" asc

View File

@@ -487,6 +487,7 @@ class AssetFaceSync extends BaseSync {
])
.leftJoin('asset', 'asset.id', 'asset_face.assetId')
.where('asset.ownerId', '=', options.userId)
.where('asset_face.isVisible', '=', true)
.stream();
}
}

View File

@@ -1,10 +0,0 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
// Sync query for faces was incorrect on server <=2.6.2
await sql`DELETE FROM session_sync_checkpoint WHERE type in ('AssetFaceV1', 'AssetFaceV2')`.execute(db);
}
export async function down(): Promise<void> {
// Not implemented
}

View File

@@ -26,6 +26,21 @@ import { JobOf } from 'src/types';
import { mimeTypes } from 'src/utils/mime-types';
import { handlePromiseError } from 'src/utils/misc';
const createMatchers = (exclusionPatterns: string[]) => {
const supportedExtensions = mimeTypes.getSupportedFileExtensions().map((extension) => extension.toLowerCase());
const expandedPatterns = exclusionPatterns.flatMap((pattern) =>
pattern.endsWith('/**') ? [pattern, pattern.slice(0, -3)] : [pattern],
);
const excludeMatcher = picomatch(expandedPatterns, { nocase: true });
return {
isExcluded: (path: string) => excludeMatcher(path.replaceAll('\\', '/')),
isSupported: (path: string) => {
const normalizedPath = path.toLowerCase();
return supportedExtensions.some((extension) => normalizedPath.endsWith(extension));
},
};
};
@Injectable()
export class LibraryService extends BaseService {
private watchLibraries = false;
@@ -90,24 +105,24 @@ export class LibraryService extends BaseService {
this.logger.log(`Starting to watch library ${library.id} with import path(s) ${library.importPaths}`);
const matcher = picomatch(`**/*{${mimeTypes.getSupportedFileExtensions().join(',')}}`, {
nocase: true,
ignore: library.exclusionPatterns,
});
const { isExcluded, isSupported } = createMatchers(library.exclusionPatterns);
let _resolve: () => void;
const ready$ = new Promise<void>((resolve) => (_resolve = resolve));
const handler = async (event: string, path: string) => {
if (matcher(path)) {
this.logger.debug(`File ${event} event received for ${path} in library ${library.id}}`);
await this.jobRepository.queue({
name: JobName.LibrarySyncFiles,
data: { libraryId: library.id, paths: [path] },
});
} else {
const ignored = !isSupported(path);
if (ignored) {
this.logger.verbose(`Ignoring file ${event} event for ${path} in library ${library.id}`);
return;
}
this.logger.debug(`File ${event} event received for ${path} in library ${library.id}}`);
await this.jobRepository.queue({
name: JobName.LibrarySyncFiles,
data: { libraryId: library.id, paths: [path] },
});
};
const deletionHandler = async (path: string) => {
@@ -123,6 +138,7 @@ export class LibraryService extends BaseService {
{
usePolling: false,
ignoreInitial: true,
ignored: isExcluded,
awaitWriteFinish: {
stabilityThreshold: 5000,
pollInterval: 1000,

View File

@@ -30,6 +30,7 @@ import { DatabaseRepository } from 'src/repositories/database.repository';
import { EmailRepository } from 'src/repositories/email.repository';
import { EventRepository } from 'src/repositories/event.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LibraryRepository } from 'src/repositories/library.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { MapRepository } from 'src/repositories/map.repository';
@@ -405,6 +406,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
case AssetRepository:
case AssetEditRepository:
case AssetJobRepository:
case LibraryRepository:
case MemoryRepository:
case NotificationRepository:
case OcrRepository:
@@ -466,6 +468,7 @@ const newMockRepository = <T>(key: ClassConstructor<T>) => {
case AlbumRepository:
case AssetRepository:
case AssetJobRepository:
case LibraryRepository:
case ConfigRepository:
case CryptoRepository:
case MemoryRepository:

View File

@@ -0,0 +1,135 @@
import { Kysely } from 'kysely';
import { mkdtemp, rm, unlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { StorageRepository } from 'src/repositories/storage.repository';
import { DB } from 'src/schema';
import { BaseService } from 'src/services/base.service';
import { newMediumService } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
let defaultDatabase: Kysely<DB>;
const setup = (db?: Kysely<DB>) => {
const { ctx } = newMediumService(BaseService, {
database: db || defaultDatabase,
real: [],
mock: [LoggingRepository],
});
return { ctx, sut: ctx.get(StorageRepository) };
};
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
const watchForEvent = (
sut: StorageRepository,
folder: string,
event: 'add' | 'change' | 'unlink',
action: () => Promise<void>,
): Promise<string> => {
return new Promise<string>((resolve, reject) => {
const timeout = setTimeout(() => {
void close().finally(() => reject(new Error(`Timed out waiting for ${event} event`)));
}, 10_000);
const onResolve = (path: string) => {
clearTimeout(timeout);
void close().finally(() => resolve(path));
};
const onReject = (error: Error) => {
clearTimeout(timeout);
void close().finally(() => reject(error));
};
const close = sut.watch(
[folder],
{
ignoreInitial: true,
usePolling: true,
interval: 50,
},
{
onReady: () => {
void action().catch((error) => onReject(error as Error));
},
onAdd: (path) => {
if (event === 'add') {
onResolve(path);
}
},
onChange: (path) => {
if (event === 'change') {
onResolve(path);
}
},
onUnlink: (path) => {
if (event === 'unlink') {
onResolve(path);
}
},
onError: (error) => onReject(error),
},
);
});
};
describe(StorageRepository.name, () => {
describe('watch', () => {
it('should fire create (add) events', async () => {
const { sut } = setup();
const folder = await mkdtemp(join(tmpdir(), 'immich-storage-watch-add-'));
const file = join(folder, 'created.jpg');
try {
const changedPath = await watchForEvent(sut, folder, 'add', async () => {
await writeFile(file, 'created');
});
expect(changedPath).toBe(file);
} finally {
await rm(folder, { recursive: true, force: true });
}
});
it('should fire change events', async () => {
const { sut } = setup();
const folder = await mkdtemp(join(tmpdir(), 'immich-storage-watch-change-'));
const file = join(folder, 'changed.jpg');
await writeFile(file, 'before');
try {
const changedPath = await watchForEvent(sut, folder, 'change', async () => {
await writeFile(file, 'after');
});
expect(changedPath).toBe(file);
} finally {
await rm(folder, { recursive: true, force: true });
}
});
it('should fire unlink events', async () => {
const { sut } = setup();
const folder = await mkdtemp(join(tmpdir(), 'immich-storage-watch-unlink-'));
const file = join(folder, 'deleted.jpg');
await writeFile(file, 'content');
try {
const changedPath = await watchForEvent(sut, folder, 'unlink', async () => {
await unlink(file);
});
expect(changedPath).toBe(file);
} finally {
await rm(folder, { recursive: true, force: true });
}
});
});
});

View File

@@ -0,0 +1,169 @@
import { ChokidarOptions } from 'chokidar';
import { Kysely } from 'kysely';
import { JobName } from 'src/enum';
import { EventRepository } from 'src/repositories/event.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LibraryRepository } from 'src/repositories/library.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { StorageRepository, WatchEvents } from 'src/repositories/storage.repository';
import { DB } from 'src/schema';
import { LibraryService } from 'src/services/library.service';
import { newMediumService } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
let defaultDatabase: Kysely<DB>;
const setup = (db?: Kysely<DB>) => {
return newMediumService(LibraryService, {
database: db || defaultDatabase,
real: [LibraryRepository],
mock: [EventRepository, JobRepository, LoggingRepository, StorageRepository],
});
};
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
const closeWatcher = () => Promise.resolve();
const setWatchMode = (sut: LibraryService) => {
const service = sut as unknown as { lock: boolean; watchLibraries: boolean };
service.lock = true;
service.watchLibraries = true;
};
const makeMockWatcher =
(paths: { add?: string[]; change?: string[]; unlink?: string[] }) =>
(_paths: string[], options: ChokidarOptions, events: Partial<WatchEvents>): (() => Promise<void>) => {
const ignored = options.ignored as ((path: string) => boolean) | undefined;
events.onReady?.();
for (const path of paths.add ?? []) {
if (!ignored?.(path)) {
events.onAdd?.(path);
}
}
for (const path of paths.change ?? []) {
if (!ignored?.(path)) {
events.onChange?.(path);
}
}
for (const path of paths.unlink ?? []) {
if (!ignored?.(path)) {
events.onUnlink?.(path);
}
}
return closeWatcher;
};
describe(`${LibraryService.name} (watch, medium)`, () => {
it('should queue add and change events for supported files', async () => {
const { sut, ctx } = setup();
const storageRepo = ctx.getMock(StorageRepository);
const jobRepo = ctx.getMock(JobRepository);
jobRepo.queue.mockResolvedValue();
setWatchMode(sut);
const { user } = await ctx.newUser();
const library = await sut.create({
ownerId: user.id,
importPaths: ['/test-assets/temp/watcher-behavior'],
exclusionPatterns: ['**/@eaDir/**'],
});
storageRepo.watch.mockImplementation(
makeMockWatcher({
add: ['/test-assets/temp/watcher-behavior/add.png'],
change: ['/test-assets/temp/watcher-behavior/change.jpg'],
}),
);
await sut.watchAll();
expect(jobRepo.queue).toHaveBeenCalledWith({
name: JobName.LibrarySyncFiles,
data: { libraryId: library.id, paths: ['/test-assets/temp/watcher-behavior/add.png'] },
});
expect(jobRepo.queue).toHaveBeenCalledWith({
name: JobName.LibrarySyncFiles,
data: { libraryId: library.id, paths: ['/test-assets/temp/watcher-behavior/change.jpg'] },
});
await sut.onShutdown();
});
it('should queue unlink events for tracked files', async () => {
const { sut, ctx } = setup();
const storageRepo = ctx.getMock(StorageRepository);
const jobRepo = ctx.getMock(JobRepository);
jobRepo.queue.mockResolvedValue();
setWatchMode(sut);
const { user } = await ctx.newUser();
const library = await sut.create({
ownerId: user.id,
importPaths: ['/test-assets/temp/watcher-behavior'],
exclusionPatterns: ['**/@eaDir/**'],
});
storageRepo.watch.mockImplementation(
makeMockWatcher({
unlink: ['/test-assets/temp/watcher-behavior/delete.png'],
}),
);
await sut.watchAll();
expect(jobRepo.queue).toHaveBeenCalledWith({
name: JobName.LibraryRemoveAsset,
data: { libraryId: library.id, paths: ['/test-assets/temp/watcher-behavior/delete.png'] },
});
await sut.onShutdown();
});
it('should ignore add, change, and unlink events in excluded directories', async () => {
const { sut, ctx } = setup();
const storageRepo = ctx.getMock(StorageRepository);
const jobRepo = ctx.getMock(JobRepository);
jobRepo.queue.mockResolvedValue();
setWatchMode(sut);
await ctx.newUser().then(({ user }) =>
sut.create({
ownerId: user.id,
importPaths: ['/test-assets/temp/watcher-behavior'],
exclusionPatterns: ['**/@eaDir/**'],
}),
);
storageRepo.watch.mockImplementation(
makeMockWatcher({
add: ['/test-assets/temp/watcher-behavior/@eaDir/add.png'],
change: ['/test-assets/temp/watcher-behavior/@eaDir/change.png'],
unlink: ['/test-assets/temp/watcher-behavior/@eaDir/unlink.png'],
}),
);
await sut.watchAll();
expect(jobRepo.queue).not.toHaveBeenCalledWith(
expect.objectContaining({
name: JobName.LibrarySyncFiles,
}),
);
expect(jobRepo.queue).not.toHaveBeenCalledWith(
expect.objectContaining({
name: JobName.LibraryRemoveAsset,
}),
);
await sut.onShutdown();
});
});

View File

@@ -23,7 +23,7 @@
{ label: '2:3', value: '2:3', width: 16, height: 24 },
{ label: '16:9', value: '16:9', width: 24, height: 14 },
{ label: '9:16', value: '9:16', width: 14, height: 24 },
{ label: 'Square', value: '1:1', width: 20, height: 20 },
{ label: $t('crop_aspect_ratio_square'), value: '1:1', width: 20, height: 20 },
];
let isRotated = $derived(transformManager.normalizedRotation % 180 !== 0);