diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b9f7fcaf5..9ca01fcf27 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,7 +9,7 @@ overrides: canvas: 2.11.2 sharp: ^0.34.5 -packageExtensionsChecksum: sha256-3l4AQg4iuprBDup+q+2JaPvbPg/7XodWCE0ZteH+s54= +packageExtensionsChecksum: sha256-vheqqqBU5SU8N8ma3OjzLM07nd511Xmy+mOvgxie+Ts= pnpmfileChecksum: sha256-AG/qwrPNpmy9q60PZwCpecoYVptglTHgH+N6RKQHOM0= @@ -421,6 +421,9 @@ importers: bcrypt: specifier: ^6.0.0 version: 6.0.0 + better-sqlite3: + specifier: ^12.6.2 + version: 12.6.2 body-parser: specifier: ^2.2.0 version: 2.2.2 @@ -605,6 +608,9 @@ importers: '@types/bcrypt': specifier: ^6.0.0 version: 6.0.0 + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 '@types/body-parser': specifier: ^1.19.6 version: 1.19.6 @@ -5080,6 +5086,9 @@ packages: '@types/bcrypt@6.0.0': resolution: {integrity: sha512-/oJGukuH3D2+D+3H4JWLaAsJ/ji86dhRidzZ/Od7H/i8g+aCmvkeCc6Ni/f9uxGLSQVCRZkX2/lqEFG2BvWtlQ==} + '@types/better-sqlite3@7.6.13': + resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} @@ -6052,6 +6061,10 @@ packages: resolution: {integrity: sha512-cU8v/EGSrnH+HnxV2z0J7/blxH8gq7Xh2JFT6Aroax7UohdmiJJlxApMxtKfuI7z68NvvVcmR78k2LbT6efhRg==} engines: {node: '>= 18'} + better-sqlite3@12.6.2: + resolution: {integrity: sha512-8VYKM3MjCa9WcaSAI3hzwhmyHVlH8tiGFwf0RlTsZPWJ1I5MkzjiudCo4KC4DxOaL/53A5B1sI/IbldNFDbsKA==} + engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x} + big.js@5.2.2: resolution: {integrity: sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==} @@ -6059,6 +6072,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bits-ui@2.14.4: resolution: {integrity: sha512-W6kenhnbd/YVvur+DKkaVJ6GldE53eLewur5AhUCqslYQ0vjZr8eWlOfwZnMiPB+PF5HMVqf61vXBvmyrAmPWg==} engines: {node: '>=20'} @@ -7585,6 +7601,10 @@ packages: resolution: {integrity: sha512-CpNH1FAhIQG5AlKndlTf05mNbuFxINyzG9629ZI/CKwr+39zWo8swxpnXc3GUfUvUfxkCCxumDPy2QVmi3XJkQ==} engines: {node: '>=20.0.0'} + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -7700,6 +7720,9 @@ packages: resolution: {integrity: sha512-8kPJMIGz1Yt/aPEwOsrR97ZyZaD1Iqm8PClb1nYFclUCkBi0Ma5IsYNQzvSFS9ib51lWyIw5mIT9rWzI/xjpzA==} engines: {node: '>=20'} + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -7900,6 +7923,9 @@ packages: get-tsconfig@4.13.0: resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + github-slugger@1.5.0: resolution: {integrity: sha512-wIh+gKBI9Nshz2o46B0B3f5k/W+WI9ZAv6y5Dn5WJ5SK1t0TnDimB4WE5rmTD05ZAIn8HALCZVmCsvj0w0v0lw==} @@ -9531,6 +9557,9 @@ packages: engines: {node: ^18 || >=20} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + natural-compare-lite@1.4.0: resolution: {integrity: sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==} @@ -9593,6 +9622,10 @@ packages: no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} + node-abi@3.87.0: + resolution: {integrity: sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ==} + engines: {node: '>=10'} + node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} @@ -10539,6 +10572,11 @@ packages: potpack@2.1.0: resolution: {integrity: sha512-pcaShQc1Shq0y+E7GqJqvZj8DTthWV1KeHGdi0Z6IAin2Oi3JnLCOfwnCo84qc+HAp52wT9nK9H7FAJp5a44GQ==} + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + hasBin: true + prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -11245,6 +11283,9 @@ packages: simple-get@3.1.1: resolution: {integrity: sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==} + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-icons@15.22.0: resolution: {integrity: sha512-i/w5Ie4tENfGYbdCo2iJ+oies0vOFd8QXWHopKOUzudfLCvnmeheF2PpHp89Z2azpc+c2su3lMiWO/SpP+429A==} engines: {node: '>=0.12.18'} @@ -11941,6 +11982,9 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + tweetnacl@0.14.5: resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} @@ -17855,6 +17899,10 @@ snapshots: dependencies: '@types/node': 24.10.9 + '@types/better-sqlite3@7.6.13': + dependencies: + '@types/node': 24.10.9 + '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 @@ -19039,10 +19087,23 @@ snapshots: transitivePeerDependencies: - supports-color + better-sqlite3@12.6.2: + dependencies: + bindings: 1.5.0 + node-addon-api: 8.5.0 + node-gyp: 12.1.0 + prebuild-install: 7.1.3 + transitivePeerDependencies: + - supports-color + big.js@5.2.2: {} binary-extensions@2.3.0: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bits-ui@2.14.4(@internationalized/date@3.10.0)(@sveltejs/kit@2.49.5(@opentelemetry/api@1.9.0)(@sveltejs/vite-plugin-svelte@6.2.4(svelte@5.48.0)(vite@7.3.1(@types/node@25.0.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.48.0)(typescript@5.9.3)(vite@7.3.1(@types/node@25.0.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.48.0): dependencies: '@floating-ui/core': 1.7.3 @@ -20808,6 +20869,8 @@ snapshots: optionalDependencies: exiftool-vendored.exe: 13.45.0 + expand-template@2.0.3: {} + expect-type@1.3.0: {} exponential-backoff@3.1.3: {} @@ -20985,6 +21048,8 @@ snapshots: transitivePeerDependencies: - supports-color + file-uri-to-path@1.0.0: {} + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -21201,6 +21266,8 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} + github-slugger@1.5.0: {} gl-matrix@3.4.4: {} @@ -23316,6 +23383,8 @@ snapshots: nanoid@5.1.6: {} + napi-build-utils@2.0.0: {} + natural-compare-lite@1.4.0: {} natural-compare@1.4.0: {} @@ -23380,6 +23449,10 @@ snapshots: lower-case: 2.0.2 tslib: 2.8.1 + node-abi@3.87.0: + dependencies: + semver: 7.7.3 + node-abort-controller@3.1.1: {} node-addon-api@4.3.0: {} @@ -24368,6 +24441,21 @@ snapshots: potpack@2.1.0: {} + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.87.0 + pump: 3.0.3 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.4 + tunnel-agent: 0.6.0 + prelude-ls@1.2.1: {} prettier-linter-helpers@1.0.1: @@ -25304,8 +25392,7 @@ snapshots: signal-exit@4.1.0: {} - simple-concat@1.0.1: - optional: true + simple-concat@1.0.1: {} simple-get@3.1.1: dependencies: @@ -25314,6 +25401,12 @@ snapshots: simple-concat: 1.0.1 optional: true + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + simple-icons@15.22.0: {} simple-icons@16.4.0: {} @@ -26150,6 +26243,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + tweetnacl@0.14.5: {} type-check@0.4.0: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index be30451965..d215fb59db 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -29,6 +29,7 @@ onlyBuiltDependencies: - sharp - '@tailwindcss/oxide' - bcrypt + - better-sqlite3 overrides: canvas: 2.11.2 sharp: ^0.34.5 @@ -59,6 +60,10 @@ packageExtensions: dependencies: node-addon-api: '*' node-gyp: '*' + better-sqlite3: + dependencies: + node-addon-api: '*' + node-gyp: '*' dedupePeerDependents: false preferWorkspacePackages: true injectWorkspacePackages: true diff --git a/server/bin/immich-dev b/server/bin/immich-dev index 84c5eea8da..88e8e12ce8 100755 --- a/server/bin/immich-dev +++ b/server/bin/immich-dev @@ -6,4 +6,4 @@ if [[ "$IMMICH_ENV" == "production" ]]; then fi cd /usr/src/app || exit -pnpm --filter immich exec nest start --debug "0.0.0.0:9230" --watch -- "$@" +exec pnpm --filter immich exec nest start --no-shell --debug "0.0.0.0:9230" --watch -- "$@" diff --git a/server/package.json b/server/package.json index c9e2c2ac22..7fc0f4bf2d 100644 --- a/server/package.json +++ b/server/package.json @@ -61,6 +61,7 @@ "archiver": "^7.0.0", "async-lock": "^1.4.0", "bcrypt": "^6.0.0", + "better-sqlite3": "^12.6.2", "body-parser": "^2.2.0", "bullmq": "^5.51.0", "chokidar": "^4.0.3", @@ -124,6 +125,7 @@ "@types/archiver": "^7.0.0", "@types/async-lock": "^1.4.2", "@types/bcrypt": "^6.0.0", + "@types/better-sqlite3": "^7.6.13", "@types/body-parser": "^1.19.6", "@types/compression": "^1.7.5", "@types/cookie-parser": "^1.4.8", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 7d622ea23d..1681d23678 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -89,8 +89,10 @@ export class BaseModule implements OnModuleInit, OnModuleDestroy { } async onModuleDestroy() { + console.log(`[${this.worker}] onModuleDestroy called - emitting AppShutdown`); await this.eventRepository.emit('AppShutdown'); await teardownTelemetry(); + console.log(`[${this.worker}] onModuleDestroy complete`); } } @@ -148,6 +150,8 @@ export class ImmichAdminModule implements OnModuleDestroy { constructor(private service: CliService) {} async onModuleDestroy() { + console.log('[ImmichAdmin] onModuleDestroy called'); await this.service.cleanup(); + console.log('[ImmichAdmin] onModuleDestroy complete'); } } diff --git a/server/src/commands/index.ts b/server/src/commands/index.ts index 2aef2e8c8b..e1d0dbf311 100644 --- a/server/src/commands/index.ts +++ b/server/src/commands/index.ts @@ -6,6 +6,10 @@ import { PromptConfirmMoveQuestions, PromptMediaLocationQuestions, } from 'src/commands/media-location.command'; +import { + MigrateThumbnailsCommand, + PromptConfirmMigrationQuestion, +} from 'src/commands/migrate-thumbnails.command'; import { DisableOAuthLogin, EnableOAuthLogin } from 'src/commands/oauth-login'; import { DisablePasswordLoginCommand, EnablePasswordLoginCommand } from 'src/commands/password-login'; import { PromptPasswordQuestions, ResetAdminPasswordCommand } from 'src/commands/reset-admin-password.command'; @@ -28,4 +32,6 @@ export const commandsAndQuestions = [ ChangeMediaLocationCommand, PromptMediaLocationQuestions, PromptConfirmMoveQuestions, + MigrateThumbnailsCommand, + PromptConfirmMigrationQuestion, ]; diff --git a/server/src/commands/migrate-thumbnails.command.ts b/server/src/commands/migrate-thumbnails.command.ts new file mode 100644 index 0000000000..2033ec9255 --- /dev/null +++ b/server/src/commands/migrate-thumbnails.command.ts @@ -0,0 +1,88 @@ +import { Command, CommandRunner, InquirerService, Option, Question, QuestionSet } from 'nest-commander'; +import { CliService } from 'src/services/cli.service'; + +@Command({ + name: 'migrate-thumbnails-to-sqlite', + description: 'Migrate thumbnails from filesystem to SQLite storage', +}) +export class MigrateThumbnailsCommand extends CommandRunner { + constructor( + private service: CliService, + private inquirer: InquirerService, + ) { + super(); + } + + @Option({ + flags: '-p, --path ', + description: 'Absolute path to the SQLite database file', + }) + parsePath(value: string) { + return value; + } + + @Option({ + flags: '-y, --yes', + description: 'Skip confirmation prompt', + }) + parseYes() { + return true; + } + + async run(passedParams: string[], options: { path?: string; yes?: boolean }): Promise { + try { + const sqlitePath = options.path ?? this.service.getDefaultThumbnailStoragePath(); + + console.log(`\nMigration settings:`); + console.log(` SQLite path: ${sqlitePath}`); + console.log(`\nThis will read all thumbnail files from the filesystem and store them in SQLite.`); + console.log(`Existing entries in SQLite will be skipped.\n`); + + if (!options.yes) { + const { confirmed } = await this.inquirer.ask<{ confirmed: boolean }>('prompt-confirm-migration', {}); + if (!confirmed) { + console.log('Migration cancelled.'); + return; + } + } + + console.log('\nStarting migration...\n'); + + let lastProgressUpdate = 0; + + const result = await this.service.migrateThumbnailsToSqlite({ + sqlitePath, + onProgress: ({ current, migrated, skipped, errors }) => { + const now = Date.now(); + if (now - lastProgressUpdate > 500 || current === 1) { + lastProgressUpdate = now; + process.stdout.write( + `\rProcessed: ${current} | Migrated: ${migrated} | Skipped: ${skipped} | Errors: ${errors}`, + ); + } + }, + }); + + console.log(`\n\nMigration complete!`); + console.log(` Total processed: ${result.total}`); + console.log(` Migrated: ${result.migrated}`); + console.log(` Skipped: ${result.skipped}`); + console.log(` Errors: ${result.errors}`); + + } catch (error) { + console.error(error); + console.error('Migration failed.'); + } + } +} + +@QuestionSet({ name: 'prompt-confirm-migration' }) +export class PromptConfirmMigrationQuestion { + @Question({ + message: 'Do you want to proceed with the migration? [Y/n]', + name: 'confirmed', + }) + parseConfirmed(value: string): boolean { + return ['yes', 'y', ''].includes((value || 'y').toLowerCase()); + } +} diff --git a/server/src/controllers/asset-media.controller.ts b/server/src/controllers/asset-media.controller.ts index ec6083cfa8..6005f065e8 100644 --- a/server/src/controllers/asset-media.controller.ts +++ b/server/src/controllers/asset-media.controller.ts @@ -42,7 +42,7 @@ import { FileUploadInterceptor, getFiles } from 'src/middleware/file-upload.inte import { LoggingRepository } from 'src/repositories/logging.repository'; import { AssetMediaService } from 'src/services/asset-media.service'; import { UploadFiles } from 'src/types'; -import { ImmichFileResponse, sendFile } from 'src/utils/file'; +import { ImmichBufferResponse, ImmichFileResponse, sendBuffer, sendFile } from 'src/utils/file'; import { FileNotEmptyValidator, UUIDParamDto } from 'src/validation'; @ApiTags(ApiTag.Assets) @@ -163,26 +163,29 @@ export class AssetMediaController { if (viewThumbnailRes instanceof ImmichFileResponse) { await sendFile(res, next, () => Promise.resolve(viewThumbnailRes), this.logger); - } else { - // viewThumbnailRes is a AssetMediaRedirectResponse - // which redirects to the original asset or a specific size to make better use of caching - const { targetSize } = viewThumbnailRes; - const [reqPath, reqSearch] = req.url.split('?'); - let redirPath: string; - const redirSearchParams = new URLSearchParams(reqSearch); - if (targetSize === 'original') { - // relative path to this.downloadAsset - redirPath = 'original'; - redirSearchParams.delete('size'); - } else if (Object.values(AssetMediaSize).includes(targetSize)) { - redirPath = reqPath; - redirSearchParams.set('size', targetSize); - } else { - throw new Error('Invalid targetSize: ' + targetSize); - } - const finalRedirPath = redirPath + '?' + redirSearchParams.toString(); - return res.redirect(finalRedirPath); + return; } + + if (viewThumbnailRes instanceof ImmichBufferResponse) { + await sendBuffer(res, next, () => Promise.resolve(viewThumbnailRes), this.logger); + return; + } + + const { targetSize } = viewThumbnailRes; + const [reqPath, reqSearch] = req.url.split('?'); + let redirPath: string; + const redirSearchParams = new URLSearchParams(reqSearch); + if (targetSize === 'original') { + redirPath = 'original'; + redirSearchParams.delete('size'); + } else if (Object.values(AssetMediaSize).includes(targetSize)) { + redirPath = reqPath; + redirSearchParams.set('size', targetSize); + } else { + throw new Error('Invalid targetSize: ' + targetSize); + } + const finalRedirPath = redirPath + '?' + redirSearchParams.toString(); + return res.redirect(finalRedirPath); } @Get(':id/video/playback') diff --git a/server/src/cores/storage.core.ts b/server/src/cores/storage.core.ts index c6821404dc..8c5d18008c 100644 --- a/server/src/cores/storage.core.ts +++ b/server/src/cores/storage.core.ts @@ -120,6 +120,10 @@ export class StorageCore { ); } + static getThumbnailStoragePath(): string { + return join(StorageCore.getMediaLocation(), 'thumbnails.sqlite3'); + } + static getEncodedVideoPath(asset: ThumbnailPathEntity) { return StorageCore.getNestedPath(StorageFolder.EncodedVideo, asset.ownerId, `${asset.id}.mp4`); } diff --git a/server/src/dtos/env.dto.ts b/server/src/dtos/env.dto.ts index e088a33413..94cb8c603b 100644 --- a/server/src/dtos/env.dto.ts +++ b/server/src/dtos/env.dto.ts @@ -140,6 +140,10 @@ export class EnvDto { @Optional() IMMICH_WORKERS_EXCLUDE?: string; + @Optional() + @Matches(/^\//, { message: 'IMMICH_THUMBNAIL_SQLITE_PATH must be an absolute path' }) + IMMICH_THUMBNAIL_SQLITE_PATH?: string; + @IsString() @Optional() DB_DATABASE_NAME?: string; diff --git a/server/src/main.ts b/server/src/main.ts index a8e3178a43..bdb2b7329b 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -25,10 +25,16 @@ class Workers { */ restarting = false; + /** + * Whether we're in graceful shutdown + */ + shuttingDown = false; + /** * Boot all enabled workers */ async bootstrap() { + this.setupSignalHandlers(); const isMaintenanceMode = await this.isMaintenanceMode(); const { workers } = new ConfigRepository().getEnv(); @@ -114,7 +120,12 @@ class Workers { } else { const worker = new Worker(workerFile); - kill = async () => void (await worker.terminate()); + kill = () => { + // Post a shutdown message to allow graceful cleanup + worker.postMessage({ type: 'shutdown' }); + // Force terminate after timeout if worker doesn't exit + setTimeout(() => void worker.terminate(), 5000); + }; anyWorker = worker; } @@ -124,17 +135,53 @@ class Workers { this.workers[name] = { kill }; } + private setupSignalHandlers() { + const shutdown = async (signal: NodeJS.Signals) => { + if (this.shuttingDown) { + return; + } + this.shuttingDown = true; + console.log(`Received ${signal}, initiating graceful shutdown...`); + + const workerNames = Object.keys(this.workers) as ImmichWorker[]; + for (const name of workerNames) { + console.log(`Sending ${signal} to ${name} worker`); + await this.workers[name]?.kill(signal); + } + + // Give workers time to shutdown gracefully + setTimeout(() => { + console.log('Shutdown timeout reached, forcing exit'); + process.exit(0); + }, 10_000); + }; + + process.on('SIGTERM', () => void shutdown('SIGTERM')); + process.on('SIGINT', () => void shutdown('SIGINT')); + } + onError(name: ImmichWorker, error: Error) { console.error(`${name} worker error: ${error}, stack: ${error.stack}`); } onExit(name: ImmichWorker, exitCode: number | null) { + console.log(`${name} worker exited with code ${exitCode}`); + delete this.workers[name]; + + // graceful shutdown in progress + if (this.shuttingDown) { + if (Object.keys(this.workers).length === 0) { + console.log('All workers have exited, shutting down main process'); + process.exit(0); + } + return; + } + // restart immich server if (exitCode === ExitCode.AppRestart || this.restarting) { this.restarting = true; console.info(`${name} worker shutdown for restart`); - delete this.workers[name]; // once all workers shut down, bootstrap again if (Object.keys(this.workers).length === 0) { @@ -145,11 +192,9 @@ class Workers { return; } - // shutdown the entire process - delete this.workers[name]; - + // unexpected exit - shutdown the entire process if (exitCode !== 0) { - console.error(`${name} worker exited with code ${exitCode}`); + console.error(`${name} worker exited unexpectedly`); if (this.workers[ImmichWorker.Api] && name !== ImmichWorker.Api) { console.error('Killing api process'); diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 1608f7b6f6..5fbbf23a10 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -423,4 +423,13 @@ export class AssetJobRepository { streamForMigrationJob() { return this.db.selectFrom('asset').select(['id']).where('asset.deletedAt', 'is', null).stream(); } + + @GenerateSql({ params: [], stream: true }) + streamAllThumbnailFiles() { + return this.db + .selectFrom('asset_file') + .select(['asset_file.assetId', 'asset_file.type', 'asset_file.path', 'asset_file.isEdited']) + .where('asset_file.type', 'in', [AssetFileType.Thumbnail, AssetFileType.Preview]) + .stream(); + } } diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 54a5d1987f..6991390da5 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -107,6 +107,10 @@ export interface EnvData { mediaLocation?: string; }; + thumbnailStorage: { + sqlitePath?: string; + }; + workers: ImmichWorker[]; plugins: { @@ -331,6 +335,10 @@ const getEnv = (): EnvData => { mediaLocation: dto.IMMICH_MEDIA_LOCATION, }, + thumbnailStorage: { + sqlitePath: dto.IMMICH_THUMBNAIL_SQLITE_PATH, + }, + telemetry: { apiPort: dto.IMMICH_API_METRICS_PORT || 8081, microservicesPort: dto.IMMICH_MICROSERVICES_METRICS_PORT || 8082, diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index 361a2e7179..5a509bafe2 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -44,6 +44,7 @@ import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; +import { ThumbnailStorageRepository } from 'src/repositories/thumbnail-storage.repository'; import { TrashRepository } from 'src/repositories/trash.repository'; import { UserRepository } from 'src/repositories/user.repository'; import { VersionHistoryRepository } from 'src/repositories/version-history.repository'; @@ -98,6 +99,7 @@ export const repositories = [ SystemMetadataRepository, TagRepository, TelemetryRepository, + ThumbnailStorageRepository, TrashRepository, UserRepository, ViewRepository, diff --git a/server/src/repositories/media.repository.spec.ts b/server/src/repositories/media.repository.spec.ts index a5380852ee..56983a8351 100644 --- a/server/src/repositories/media.repository.spec.ts +++ b/server/src/repositories/media.repository.spec.ts @@ -2,7 +2,7 @@ import sharp from 'sharp'; import { AssetFace } from 'src/database'; import { AssetEditAction, MirrorAxis } from 'src/dtos/editing.dto'; import { AssetOcrResponseDto } from 'src/dtos/ocr.dto'; -import { SourceType } from 'src/enum'; +import { ImageFormat, SourceType } from 'src/enum'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { BoundingBox } from 'src/repositories/machine-learning.repository'; import { MediaRepository } from 'src/repositories/media.repository'; @@ -664,4 +664,51 @@ describe(MediaRepository.name, () => { }); }); }); + + describe('generateThumbnailToBuffer', () => { + it('should return a buffer instead of writing to file', async () => { + const inputBuffer = await sharp({ + create: { width: 100, height: 100, channels: 3, background: { r: 255, g: 0, b: 0 } }, + }) + .png() + .toBuffer(); + + const result = await sut.generateThumbnailToBuffer(inputBuffer, { + format: ImageFormat.Webp, + quality: 80, + size: 50, + colorspace: 'srgb', + processInvalidImages: false, + }); + + expect(result).toBeInstanceOf(Buffer); + expect(result.length).toBeGreaterThan(0); + + const metadata = await sharp(result).metadata(); + expect(metadata.format).toBe('webp'); + expect(metadata.width).toBeLessThanOrEqual(50); + }); + + it('should apply same options as generateThumbnail', async () => { + const inputBuffer = await sharp({ + create: { width: 200, height: 200, channels: 3, background: { r: 0, g: 255, b: 0 } }, + }) + .png() + .toBuffer(); + + const result = await sut.generateThumbnailToBuffer(inputBuffer, { + format: ImageFormat.Jpeg, + quality: 90, + size: 75, + colorspace: 'srgb', + processInvalidImages: false, + progressive: true, + }); + + const metadata = await sharp(result).metadata(); + expect(metadata.format).toBe('jpeg'); + expect(metadata.width).toBe(75); + expect(metadata.height).toBe(75); + }); + }); }); diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 33025e73cf..a3164d4a74 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -182,6 +182,17 @@ export class MediaRepository { await decoded.toFile(output); } + async generateThumbnailToBuffer(input: string | Buffer, options: GenerateThumbnailOptions): Promise { + const pipeline = await this.getImageDecodingPipeline(input, options); + return pipeline + .toFormat(options.format, { + quality: options.quality, + chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0', + progressive: options.progressive, + }) + .toBuffer(); + } + private async getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) { let pipeline = sharp(input, { // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes diff --git a/server/src/repositories/thumbnail-storage.repository.spec.ts b/server/src/repositories/thumbnail-storage.repository.spec.ts new file mode 100644 index 0000000000..d7c9984346 --- /dev/null +++ b/server/src/repositories/thumbnail-storage.repository.spec.ts @@ -0,0 +1,305 @@ +import Database from 'better-sqlite3'; +import { randomUUID } from 'node:crypto'; +import { existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AssetFileType } from 'src/enum'; +import { LoggingRepository } from 'src/repositories/logging.repository'; +import { ThumbnailStorageRepository } from 'src/repositories/thumbnail-storage.repository'; +import { automock } from 'test/utils'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +describe(ThumbnailStorageRepository.name, () => { + let sut: ThumbnailStorageRepository; + let testDbPath: string; + + beforeEach(() => { + testDbPath = join(tmpdir(), `immich-test-thumbnails-${randomUUID()}.db`); + const logger = automock(LoggingRepository, { args: [undefined, { getEnv: () => ({}) }], strict: false }); + sut = new ThumbnailStorageRepository(logger); + }); + + afterEach(() => { + sut.close(); + if (existsSync(testDbPath)) { + rmSync(testDbPath); + } + }); + + describe('initialize', () => { + it('should create database and schema', () => { + sut.initialize(testDbPath); + + expect(sut.isEnabled()).toBe(true); + + const db = new Database(testDbPath, { readonly: true }); + const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='thumbnails'").all(); + db.close(); + + expect(tables).toHaveLength(1); + }); + + it('should enable WAL mode', () => { + sut.initialize(testDbPath); + + const db = new Database(testDbPath, { readonly: true }); + const result = db.prepare('PRAGMA journal_mode').get() as { journal_mode: string }; + db.close(); + + expect(result.journal_mode).toBe('wal'); + }); + }); + + describe('store', () => { + const assetId = randomUUID(); + const testData = Buffer.from('test thumbnail data'); + const mimeType = 'image/webp'; + + beforeEach(() => { + sut.initialize(testDbPath); + }); + + it('should store thumbnail data', async () => { + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: false, + data: testData, + mimeType, + }); + + const result = await sut.get(assetId, AssetFileType.Thumbnail, false); + expect(result).not.toBeNull(); + expect(result!.data).toEqual(testData); + expect(result!.mimeType).toBe(mimeType); + expect(result!.size).toBe(testData.length); + }); + + it('should replace existing thumbnail on conflict', async () => { + const newData = Buffer.from('updated thumbnail data'); + + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: false, + data: testData, + mimeType, + }); + + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: false, + data: newData, + mimeType, + }); + + const result = await sut.get(assetId, AssetFileType.Thumbnail, false); + expect(result!.data).toEqual(newData); + expect(result!.size).toBe(newData.length); + }); + + it('should store with correct mime type and size', async () => { + const jpegData = Buffer.from('jpeg thumbnail data with different size'); + const jpegMimeType = 'image/jpeg'; + + await sut.store({ + assetId, + type: AssetFileType.Preview, + isEdited: false, + data: jpegData, + mimeType: jpegMimeType, + }); + + const result = await sut.get(assetId, AssetFileType.Preview, false); + expect(result!.mimeType).toBe(jpegMimeType); + expect(result!.size).toBe(jpegData.length); + }); + }); + + describe('get', () => { + const assetId = randomUUID(); + const testData = Buffer.from('test thumbnail data'); + const mimeType = 'image/webp'; + + beforeEach(async () => { + sut.initialize(testDbPath); + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: false, + data: testData, + mimeType, + }); + }); + + it('should return stored thumbnail data', async () => { + const result = await sut.get(assetId, AssetFileType.Thumbnail, false); + + expect(result).not.toBeNull(); + expect(result!.data).toEqual(testData); + expect(result!.mimeType).toBe(mimeType); + }); + + it('should return null for non-existent thumbnail', async () => { + const result = await sut.get(randomUUID(), AssetFileType.Thumbnail, false); + + expect(result).toBeNull(); + }); + + it('should distinguish between edited and non-edited', async () => { + const editedData = Buffer.from('edited thumbnail data'); + + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: true, + data: editedData, + mimeType, + }); + + const nonEditedResult = await sut.get(assetId, AssetFileType.Thumbnail, false); + const editedResult = await sut.get(assetId, AssetFileType.Thumbnail, true); + + expect(nonEditedResult!.data).toEqual(testData); + expect(editedResult!.data).toEqual(editedData); + }); + + it('should distinguish between different thumbnail types', async () => { + const previewData = Buffer.from('preview data'); + + await sut.store({ + assetId, + type: AssetFileType.Preview, + isEdited: false, + data: previewData, + mimeType, + }); + + const thumbnailResult = await sut.get(assetId, AssetFileType.Thumbnail, false); + const previewResult = await sut.get(assetId, AssetFileType.Preview, false); + + expect(thumbnailResult!.data).toEqual(testData); + expect(previewResult!.data).toEqual(previewData); + }); + + it('should fall back to non-edited when edited is requested but not found', async () => { + const result = await sut.get(assetId, AssetFileType.Thumbnail, true); + + expect(result).not.toBeNull(); + expect(result!.data).toEqual(testData); + }); + + it('should return edited when both exist and edited is requested', async () => { + const editedData = Buffer.from('edited thumbnail data'); + + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: true, + data: editedData, + mimeType, + }); + + const result = await sut.get(assetId, AssetFileType.Thumbnail, true); + + expect(result!.data).toEqual(editedData); + }); + + it('should not fall back to edited when non-edited is requested but not found', async () => { + const newAssetId = randomUUID(); + const editedData = Buffer.from('edited only data'); + + await sut.store({ + assetId: newAssetId, + type: AssetFileType.Thumbnail, + isEdited: true, + data: editedData, + mimeType, + }); + + const result = await sut.get(newAssetId, AssetFileType.Thumbnail, false); + + expect(result).toBeNull(); + }); + }); + + describe('delete', () => { + const assetId = randomUUID(); + const testData = Buffer.from('test thumbnail data'); + const mimeType = 'image/webp'; + + beforeEach(async () => { + sut.initialize(testDbPath); + await sut.store({ + assetId, + type: AssetFileType.Thumbnail, + isEdited: false, + data: testData, + mimeType, + }); + await sut.store({ + assetId, + type: AssetFileType.Preview, + isEdited: false, + data: testData, + mimeType, + }); + }); + + it('should delete specific thumbnail', async () => { + await sut.delete(assetId, AssetFileType.Thumbnail, false); + + const result = await sut.get(assetId, AssetFileType.Thumbnail, false); + expect(result).toBeNull(); + }); + + it('should not affect other thumbnails for same asset', async () => { + await sut.delete(assetId, AssetFileType.Thumbnail, false); + + const previewResult = await sut.get(assetId, AssetFileType.Preview, false); + expect(previewResult).not.toBeNull(); + }); + }); + + describe('deleteByAsset', () => { + const assetId = randomUUID(); + const otherAssetId = randomUUID(); + const testData = Buffer.from('test thumbnail data'); + const mimeType = 'image/webp'; + + beforeEach(async () => { + sut.initialize(testDbPath); + await sut.store({ assetId, type: AssetFileType.Thumbnail, isEdited: false, data: testData, mimeType }); + await sut.store({ assetId, type: AssetFileType.Preview, isEdited: false, data: testData, mimeType }); + await sut.store({ assetId, type: AssetFileType.Thumbnail, isEdited: true, data: testData, mimeType }); + await sut.store({ assetId: otherAssetId, type: AssetFileType.Thumbnail, isEdited: false, data: testData, mimeType }); + }); + + it('should delete all thumbnails for an asset', async () => { + await sut.deleteByAsset(assetId); + + expect(await sut.get(assetId, AssetFileType.Thumbnail, false)).toBeNull(); + expect(await sut.get(assetId, AssetFileType.Preview, false)).toBeNull(); + expect(await sut.get(assetId, AssetFileType.Thumbnail, true)).toBeNull(); + }); + + it('should not affect other assets', async () => { + await sut.deleteByAsset(assetId); + + const otherAssetResult = await sut.get(otherAssetId, AssetFileType.Thumbnail, false); + expect(otherAssetResult).not.toBeNull(); + }); + }); + + describe('isEnabled', () => { + it('should return false when database is not initialized', () => { + expect(sut.isEnabled()).toBe(false); + }); + + it('should return true when database is initialized', () => { + sut.initialize(testDbPath); + expect(sut.isEnabled()).toBe(true); + }); + }); +}); diff --git a/server/src/repositories/thumbnail-storage.repository.ts b/server/src/repositories/thumbnail-storage.repository.ts new file mode 100644 index 0000000000..f528d94e3c --- /dev/null +++ b/server/src/repositories/thumbnail-storage.repository.ts @@ -0,0 +1,160 @@ +import { Injectable, OnModuleDestroy } from '@nestjs/common'; +import Database, { Database as DatabaseType, Statement } from 'better-sqlite3'; +import { AssetFileType } from 'src/enum'; +import { LoggingRepository } from 'src/repositories/logging.repository'; + +export interface ThumbnailData { + assetId: string; + type: AssetFileType; + isEdited: boolean; + data: Buffer; + mimeType: string; +} + +export interface ThumbnailResult { + data: Buffer; + mimeType: string; + size: number; +} + +interface ThumbnailRow { + data: Buffer; + mime_type: string; + size: number; +} + +@Injectable() +export class ThumbnailStorageRepository implements OnModuleDestroy { + private database: DatabaseType | null = null; + private insertStatement: Statement | null = null; + private selectStatement: Statement | null = null; + private deleteStatement: Statement | null = null; + private deleteByAssetStatement: Statement | null = null; + + constructor(private logger: LoggingRepository) { + this.logger.setContext(ThumbnailStorageRepository.name); + } + + initialize(databasePath: string): void { + this.database = new Database(databasePath); + + this.database.pragma('page_size = 32768'); + this.database.pragma('journal_mode = WAL'); + this.database.pragma('synchronous = NORMAL'); + this.database.pragma('cache_size = -131072'); + this.database.pragma('mmap_size = 2147483648'); + this.database.pragma('temp_store = MEMORY'); + this.database.pragma('wal_autocheckpoint = 10000'); + + this.database.exec(` + CREATE TABLE IF NOT EXISTS thumbnails ( + asset_id TEXT NOT NULL, + type TEXT NOT NULL, + is_edited INTEGER NOT NULL DEFAULT 0, + data BLOB NOT NULL, + mime_type TEXT NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (asset_id, type, is_edited) + ) + `); + + this.insertStatement = this.database.prepare(` + INSERT OR REPLACE INTO thumbnails (asset_id, type, is_edited, data, mime_type, size) + VALUES (?, ?, ?, ?, ?, ?) + `); + + this.selectStatement = this.database.prepare(` + SELECT data, mime_type, size FROM thumbnails + WHERE asset_id = ? AND type = ? AND is_edited = ? + `); + + this.deleteStatement = this.database.prepare(` + DELETE FROM thumbnails WHERE asset_id = ? AND type = ? AND is_edited = ? + `); + + this.deleteByAssetStatement = this.database.prepare(` + DELETE FROM thumbnails WHERE asset_id = ? + `); + + this.logger.log(`SQLite thumbnail storage initialized at ${databasePath}`); + } + + isEnabled(): boolean { + return this.database !== null; + } + + store(thumbnail: ThumbnailData): void { + if (!this.insertStatement) { + throw new Error('SQLite thumbnail storage not initialized'); + } + + const isEditedInt = thumbnail.isEdited ? 1 : 0; + this.insertStatement.run( + thumbnail.assetId, + thumbnail.type, + isEditedInt, + thumbnail.data, + thumbnail.mimeType, + thumbnail.data.length, + ); + } + + get(assetId: string, type: AssetFileType, isEdited: boolean): ThumbnailResult | null { + if (!this.selectStatement) { + return null; + } + + const isEditedInt = isEdited ? 1 : 0; + let result = this.selectStatement.get(assetId, type, isEditedInt) as ThumbnailRow | undefined; + + if (!result && isEdited) { + result = this.selectStatement.get(assetId, type, 0) as ThumbnailRow | undefined; + } + + if (!result) { + return null; + } + + return { + data: result.data, + mimeType: result.mime_type, + size: result.size, + }; + } + + delete(assetId: string, type: AssetFileType, isEdited: boolean): void { + if (!this.deleteStatement) { + return; + } + + const isEditedInt = isEdited ? 1 : 0; + this.deleteStatement.run(assetId, type, isEditedInt); + } + + deleteByAsset(assetId: string): void { + if (!this.deleteByAssetStatement) { + return; + } + + this.deleteByAssetStatement.run(assetId); + } + + close(): void { + if (this.database) { + this.logger.log('Closing SQLite thumbnail storage database'); + this.database.pragma('wal_checkpoint(TRUNCATE)'); + this.database.close(); + this.database = null; + this.insertStatement = null; + this.selectStatement = null; + this.deleteStatement = null; + this.deleteByAssetStatement = null; + this.logger.log('SQLite thumbnail storage database closed'); + } + } + + onModuleDestroy(): void { + this.logger.log('onModuleDestroy called - closing SQLite thumbnail storage'); + this.close(); + } +} diff --git a/server/src/services/asset-media.service.spec.ts b/server/src/services/asset-media.service.spec.ts index 0bcb87e2f4..3da07e1b75 100644 --- a/server/src/services/asset-media.service.spec.ts +++ b/server/src/services/asset-media.service.spec.ts @@ -14,7 +14,7 @@ import { AuthRequest } from 'src/middleware/auth.guard'; import { AssetMediaService } from 'src/services/asset-media.service'; import { UploadBody } from 'src/types'; import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database'; -import { ImmichFileResponse } from 'src/utils/file'; +import { ImmichBufferResponse, ImmichFileResponse } from 'src/utils/file'; import { assetStub } from 'test/fixtures/asset.stub'; import { authStub } from 'test/fixtures/auth.stub'; import { fileStub } from 'test/fixtures/file.stub'; @@ -762,6 +762,128 @@ describe(AssetMediaService.name, () => { ); expect(mocks.asset.getForThumbnail).toHaveBeenCalledWith(assetStub.image.id, AssetFileType.Thumbnail, true); }); + + describe('with SQLite storage', () => { + it('should return ImmichBufferResponse when thumbnail exists in SQLite', async () => { + const thumbnailData = Buffer.from('thumbnail-data'); + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); + mocks.thumbnailStorage.isEnabled.mockReturnValue(true); + mocks.thumbnailStorage.get.mockReturnValue({ + data: thumbnailData, + mimeType: 'image/webp', + size: thumbnailData.length, + }); + + await expect( + sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.THUMBNAIL }), + ).resolves.toEqual( + new ImmichBufferResponse({ + data: thumbnailData, + contentType: 'image/webp', + cacheControl: CacheControl.PrivateWithCache, + fileName: 'asset-id_thumbnail.webp', + }), + ); + + expect(mocks.thumbnailStorage.get).toHaveBeenCalledWith(assetStub.image.id, AssetFileType.Thumbnail, false); + expect(mocks.asset.getForThumbnail).not.toHaveBeenCalled(); + }); + + it('should fall back to filesystem when thumbnail not in SQLite', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); + mocks.thumbnailStorage.isEnabled.mockReturnValue(true); + mocks.thumbnailStorage.get.mockReturnValue(null); + mocks.asset.getForThumbnail.mockResolvedValue({ + ...assetStub.image, + path: '/uploads/user-id/thumbs/path.jpg', + }); + + await expect( + sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.THUMBNAIL }), + ).resolves.toEqual( + new ImmichFileResponse({ + path: '/uploads/user-id/thumbs/path.jpg', + cacheControl: CacheControl.PrivateWithCache, + contentType: 'image/jpeg', + fileName: 'asset-id_thumbnail.jpg', + }), + ); + + expect(mocks.thumbnailStorage.get).toHaveBeenCalledWith(assetStub.image.id, AssetFileType.Thumbnail, false); + expect(mocks.asset.getForThumbnail).toHaveBeenCalled(); + }); + + it('should handle edited thumbnails from SQLite', async () => { + const thumbnailData = Buffer.from('edited-thumbnail-data'); + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); + mocks.thumbnailStorage.isEnabled.mockReturnValue(true); + mocks.thumbnailStorage.get.mockReturnValue({ + data: thumbnailData, + mimeType: 'image/webp', + size: thumbnailData.length, + }); + + await expect( + sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.THUMBNAIL, edited: true }), + ).resolves.toEqual( + new ImmichBufferResponse({ + data: thumbnailData, + contentType: 'image/webp', + cacheControl: CacheControl.PrivateWithCache, + fileName: 'asset-id_thumbnail.webp', + }), + ); + + expect(mocks.thumbnailStorage.get).toHaveBeenCalledWith(assetStub.image.id, AssetFileType.Thumbnail, true); + }); + + it('should handle preview size from SQLite', async () => { + const previewData = Buffer.from('preview-data'); + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); + mocks.thumbnailStorage.isEnabled.mockReturnValue(true); + mocks.thumbnailStorage.get.mockReturnValue({ + data: previewData, + mimeType: 'image/jpeg', + size: previewData.length, + }); + + await expect( + sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.PREVIEW }), + ).resolves.toEqual( + new ImmichBufferResponse({ + data: previewData, + contentType: 'image/jpeg', + cacheControl: CacheControl.PrivateWithCache, + fileName: 'asset-id_preview.jpg', + }), + ); + + expect(mocks.thumbnailStorage.get).toHaveBeenCalledWith(assetStub.image.id, AssetFileType.Preview, false); + }); + + it('should skip SQLite lookup when storage is disabled', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id])); + mocks.thumbnailStorage.isEnabled.mockReturnValue(false); + mocks.asset.getForThumbnail.mockResolvedValue({ + ...assetStub.image, + path: '/uploads/user-id/thumbs/path.jpg', + }); + + await expect( + sut.viewThumbnail(authStub.admin, assetStub.image.id, { size: AssetMediaSize.THUMBNAIL }), + ).resolves.toEqual( + new ImmichFileResponse({ + path: '/uploads/user-id/thumbs/path.jpg', + cacheControl: CacheControl.PrivateWithCache, + contentType: 'image/jpeg', + fileName: 'asset-id_thumbnail.jpg', + }), + ); + + expect(mocks.thumbnailStorage.get).not.toHaveBeenCalled(); + expect(mocks.asset.getForThumbnail).toHaveBeenCalled(); + }); + }); }); describe('playbackVideo', () => { diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts index 020bda4df7..10cf6c1e7b 100644 --- a/server/src/services/asset-media.service.ts +++ b/server/src/services/asset-media.service.ts @@ -37,7 +37,12 @@ import { UploadFile, UploadRequest } from 'src/types'; import { requireUploadAccess } from 'src/utils/access'; import { asUploadRequest, onBeforeLink } from 'src/utils/asset.util'; import { isAssetChecksumConstraint } from 'src/utils/database'; -import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file'; +import { + getFilenameExtension, + getFileNameWithoutExtension, + ImmichBufferResponse, + ImmichFileResponse, +} from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; import { fromChecksum } from 'src/utils/request'; @@ -219,7 +224,7 @@ export class AssetMediaService extends BaseService { auth: AuthDto, id: string, dto: AssetMediaOptionsDto, - ): Promise { + ): Promise { await this.requireAccess({ auth, permission: Permission.AssetView, ids: [id] }); if (dto.size === AssetMediaSize.Original) { @@ -231,20 +236,30 @@ export class AssetMediaService extends BaseService { } const size = (dto.size ?? AssetMediaSize.THUMBNAIL) as unknown as AssetFileType; - const { originalPath, originalFileName, path } = await this.assetRepository.getForThumbnail( - id, - size, - dto.edited ?? false, - ); + const isEdited = dto.edited ?? false; + + if (this.thumbnailStorageRepository.isEnabled()) { + const thumbnail = this.thumbnailStorageRepository.get(id, size, isEdited); + if (thumbnail) { + // this.logger.log(`Thumbnail served from SQLite: assetId=${id}, type=${size}`); + const extension = mimeTypes.toExtension(thumbnail.mimeType) || ''; + const fileName = `${id}_${size}${extension}`; + return new ImmichBufferResponse({ + data: thumbnail.data, + contentType: thumbnail.mimeType, + cacheControl: CacheControl.PrivateWithCache, + fileName, + }); + } + } + + const { originalPath, originalFileName, path } = await this.assetRepository.getForThumbnail(id, size, isEdited); if (size === AssetFileType.FullSize && mimeTypes.isWebSupportedImage(originalPath) && !dto.edited) { - // use original file for web supported images return { targetSize: 'original' }; } if (dto.size === AssetMediaSize.FULLSIZE && !path) { - // downgrade to preview if fullsize is not available. - // e.g. disabled or not yet (re)generated return { targetSize: AssetMediaSize.PREVIEW }; } diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index b3a50a07ae..3e31e3a291 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -51,6 +51,7 @@ import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; +import { ThumbnailStorageRepository } from 'src/repositories/thumbnail-storage.repository'; import { TrashRepository } from 'src/repositories/trash.repository'; import { UserRepository } from 'src/repositories/user.repository'; import { VersionHistoryRepository } from 'src/repositories/version-history.repository'; @@ -108,6 +109,7 @@ export const BASE_SERVICE_DEPENDENCIES = [ SystemMetadataRepository, TagRepository, TelemetryRepository, + ThumbnailStorageRepository, TrashRepository, UserRepository, VersionHistoryRepository, @@ -167,6 +169,7 @@ export class BaseService { protected systemMetadataRepository: SystemMetadataRepository, protected tagRepository: TagRepository, protected telemetryRepository: TelemetryRepository, + protected thumbnailStorageRepository: ThumbnailStorageRepository, protected trashRepository: TrashRepository, protected userRepository: UserRepository, protected versionRepository: VersionHistoryRepository, diff --git a/server/src/services/cli.service.ts b/server/src/services/cli.service.ts index ce62f98aa1..b45fc9650e 100644 --- a/server/src/services/cli.service.ts +++ b/server/src/services/cli.service.ts @@ -1,12 +1,14 @@ import { Injectable } from '@nestjs/common'; import { isAbsolute } from 'node:path'; import { SALT_ROUNDS } from 'src/constants'; +import { StorageCore } from 'src/cores/storage.core'; import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto'; import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto'; -import { MaintenanceAction, SystemMetadataKey } from 'src/enum'; +import { AssetFileType, MaintenanceAction, SystemMetadataKey } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance'; import { getExternalDomain } from 'src/utils/misc'; +import { mimeTypes } from 'src/utils/mime-types'; @Injectable() export class CliService extends BaseService { @@ -186,4 +188,88 @@ export class CliService extends BaseService { cleanup() { return this.databaseRepository.shutdown(); } + + getDefaultThumbnailStoragePath(): string { + const envData = this.configRepository.getEnv(); + if (envData.thumbnailStorage.sqlitePath) { + return envData.thumbnailStorage.sqlitePath; + } + + const mediaLocation = envData.storage.mediaLocation ?? this.detectMediaLocation(); + StorageCore.setMediaLocation(mediaLocation); + return StorageCore.getThumbnailStoragePath(); + } + + private detectMediaLocation(): string { + const candidates = ['/data', '/usr/src/app/upload']; + for (const candidate of candidates) { + if (this.storageRepository.existsSync(candidate)) { + return candidate; + } + } + return '/usr/src/app/upload'; + } + + async migrateThumbnailsToSqlite(options: { + sqlitePath: string; + onProgress: (progress: { current: number; migrated: number; skipped: number; errors: number }) => void; + }): Promise<{ total: number; migrated: number; skipped: number; errors: number }> { + const { sqlitePath, onProgress } = options; + + if (!isAbsolute(sqlitePath)) { + throw new Error('SQLite path must be an absolute path'); + } + + this.thumbnailStorageRepository.initialize(sqlitePath); + + let current = 0; + let migrated = 0; + let skipped = 0; + let errors = 0; + + for await (const file of this.assetJobRepository.streamAllThumbnailFiles()) { + current++; + + try { + const existingData = this.thumbnailStorageRepository.get( + file.assetId, + file.type as AssetFileType, + file.isEdited, + ); + + if (existingData) { + skipped++; + onProgress({ current, migrated, skipped, errors }); + continue; + } + + const fileExists = await this.storageRepository.checkFileExists(file.path); + if (!fileExists) { + skipped++; + onProgress({ current, migrated, skipped, errors }); + continue; + } + + const data = await this.storageRepository.readFile(file.path); + const mimeType = mimeTypes.lookup(file.path) || 'image/jpeg'; + + this.thumbnailStorageRepository.store({ + assetId: file.assetId, + type: file.type as AssetFileType, + isEdited: file.isEdited, + data, + mimeType, + }); + + migrated++; + } catch (error) { + errors++; + this.logger.error(`Failed to migrate thumbnail for asset ${file.assetId}: ${error}`); + } + + onProgress({ current, migrated, skipped, errors }); + } + + return { total: current, migrated, skipped, errors }; + } } diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index fa2607faa9..735c9099c1 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -1128,6 +1128,82 @@ describe(MediaService.name, () => { expect.stringContaining('fullsize.jpeg'), ); }); + + describe('with SQLite storage enabled', () => { + beforeEach(() => { + mocks.thumbnailStorage.isEnabled.mockReturnValue(true); + }); + + it('should store thumbnail in SQLite when enabled', async () => { + const thumbnailBuffer = Buffer.from('thumbnail-data'); + mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image); + mocks.media.generateThumbnailToBuffer.mockResolvedValue(thumbnailBuffer); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(mocks.media.generateThumbnailToBuffer).toHaveBeenCalledTimes(2); + expect(mocks.thumbnailStorage.store).toHaveBeenCalledWith({ + assetId: assetStub.image.id, + type: AssetFileType.Thumbnail, + isEdited: false, + data: thumbnailBuffer, + mimeType: 'image/webp', + }); + expect(mocks.thumbnailStorage.store).toHaveBeenCalledWith({ + assetId: assetStub.image.id, + type: AssetFileType.Preview, + isEdited: false, + data: thumbnailBuffer, + mimeType: 'image/jpeg', + }); + expect(mocks.media.generateThumbnail).not.toHaveBeenCalled(); + }); + + it('should not write to filesystem when SQLite is enabled', async () => { + const thumbnailBuffer = Buffer.from('thumbnail-data'); + mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image); + mocks.media.generateThumbnailToBuffer.mockResolvedValue(thumbnailBuffer); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(mocks.media.generateThumbnail).not.toHaveBeenCalled(); + expect(mocks.asset.upsertFiles).not.toHaveBeenCalled(); + }); + + it('should store with correct mime type for JPEG preview', async () => { + const thumbnailBuffer = Buffer.from('preview-data'); + mocks.systemMetadata.get.mockResolvedValue({ + image: { preview: { format: ImageFormat.Jpeg } }, + }); + mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image); + mocks.media.generateThumbnailToBuffer.mockResolvedValue(thumbnailBuffer); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(mocks.thumbnailStorage.store).toHaveBeenCalledWith( + expect.objectContaining({ + type: AssetFileType.Preview, + mimeType: 'image/jpeg', + }), + ); + }); + }); + + describe('with SQLite storage disabled', () => { + beforeEach(() => { + mocks.thumbnailStorage.isEnabled.mockReturnValue(false); + }); + + it('should continue using filesystem when SQLite is disabled', async () => { + mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(mocks.media.generateThumbnail).toHaveBeenCalled(); + expect(mocks.media.generateThumbnailToBuffer).not.toHaveBeenCalled(); + expect(mocks.thumbnailStorage.store).not.toHaveBeenCalled(); + }); + }); }); describe('handleAssetEditThumbnailGeneration', () => { diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index b9b8d74737..d1ededd19d 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -4,7 +4,7 @@ import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { ImagePathOptions, StorageCore, ThumbnailPathEntity } from 'src/cores/storage.core'; import { AssetFile, Exif } from 'src/database'; import { OnEvent, OnJob } from 'src/decorators'; -import { AssetEditAction, CropParameters } from 'src/dtos/editing.dto'; +import { AssetEditAction, AssetEditActionItem, CropParameters } from 'src/dtos/editing.dto'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; import { AssetFileType, @@ -35,6 +35,7 @@ import { ImageDimensions, JobItem, JobOf, + RawImageInfo, VideoFormat, VideoInterfaces, VideoStreamInfo, @@ -56,6 +57,9 @@ interface UpsertFileOptions { type ThumbnailAsset = NonNullable>>; +// Feature flag: Enable fullsize thumbnail storage in SQLite +const SQLITE_STORE_FULLSIZE = true; + @Injectable() export class MediaService extends BaseService { videoInterfaces: VideoInterfaces = { dri: [], mali: false }; @@ -320,8 +324,27 @@ export class MediaService extends BaseService { const extractedImage = await this.extractOriginalImage(asset, image, useEdits); const { info, data, colorspace, generateFullsize, convertFullsize, extracted } = extractedImage; - // generate final images const thumbnailOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] }; + const decodedDimensions = { width: info.width, height: info.height }; + const fullsizeDimensions = useEdits ? getOutputDimensions(asset.edits, decodedDimensions) : decodedDimensions; + + if (this.thumbnailStorageRepository.isEnabled()) { + return this.generateImageThumbnailsToSqlite( + asset, + data, + image, + thumbnailOptions, + useEdits, + fullsizeDimensions, + extracted, + generateFullsize, + convertFullsize, + ); + } else { + console.log('not enabled'); + } + + // generate final images to filesystem const promises = [ this.mediaRepository.generateThumbhash(data, thumbnailOptions), this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailFile.path), @@ -376,9 +399,6 @@ export class MediaService extends BaseService { await Promise.all(promises); } - const decodedDimensions = { width: info.width, height: info.height }; - const fullsizeDimensions = useEdits ? getOutputDimensions(asset.edits, decodedDimensions) : decodedDimensions; - return { files: fullsizeFile ? [previewFile, thumbnailFile, fullsizeFile] : [previewFile, thumbnailFile], thumbhash: outputs[0] as Buffer, @@ -386,6 +406,82 @@ export class MediaService extends BaseService { }; } + private async generateImageThumbnailsToSqlite( + asset: ThumbnailAsset, + data: Buffer, + image: SystemConfig['image'], + thumbnailOptions: { + colorspace: Colorspace; + processInvalidImages: boolean; + raw: RawImageInfo; + edits: AssetEditActionItem[]; + }, + useEdits: boolean, + fullsizeDimensions: ImageDimensions, + extracted: { buffer: Buffer; format: RawExtractedFormat } | null, + generateFullsize: boolean, + convertFullsize: boolean, + ) { + const [thumbnailBuffer, previewBuffer, thumbhash] = await Promise.all([ + this.mediaRepository.generateThumbnailToBuffer(data, { ...image.thumbnail, ...thumbnailOptions }), + this.mediaRepository.generateThumbnailToBuffer(data, { ...image.preview, ...thumbnailOptions }), + this.mediaRepository.generateThumbhash(data, thumbnailOptions), + ]); + + // Check if fullsize should be stored in SQLite + let fullsizeBuffer: Buffer | null = null; + let fullsizeMimeType: string | null = null; + + if (SQLITE_STORE_FULLSIZE && generateFullsize) { + if (convertFullsize) { + // Convert scenario: generate fullsize from data buffer + fullsizeBuffer = await this.mediaRepository.generateThumbnailToBuffer(data, { + format: image.fullsize.format, + quality: image.fullsize.quality, + progressive: image.fullsize.progressive, + ...thumbnailOptions, + }); + fullsizeMimeType = `image/${image.fullsize.format}`; + } else if (extracted && extracted.format === RawExtractedFormat.Jpeg) { + // Extract scenario: use extracted buffer directly + fullsizeBuffer = extracted.buffer; + fullsizeMimeType = 'image/jpeg'; + } + } + + this.thumbnailStorageRepository.store({ + assetId: asset.id, + type: AssetFileType.Thumbnail, + isEdited: useEdits, + data: thumbnailBuffer, + mimeType: `image/${image.thumbnail.format}`, + }); + this.thumbnailStorageRepository.store({ + assetId: asset.id, + type: AssetFileType.Preview, + isEdited: useEdits, + data: previewBuffer, + mimeType: `image/${image.preview.format}`, + }); + + // Store fullsize if generated + if (fullsizeBuffer && fullsizeMimeType) { + this.thumbnailStorageRepository.store({ + assetId: asset.id, + type: AssetFileType.FullSize, + isEdited: useEdits, + data: fullsizeBuffer, + mimeType: fullsizeMimeType, + }); + } + + return { + files: [] as UpsertFileOptions[], + thumbhash, + fullsizeDimensions, + }; + } + @OnJob({ name: JobName.PersonGenerateThumbnail, queue: QueueName.ThumbnailGeneration }) async handleGeneratePersonThumbnail({ id }: JobOf): Promise { const { machineLearning, metadata, image } = await this.getConfig({ withCache: true }); diff --git a/server/src/services/storage.service.ts b/server/src/services/storage.service.ts index 71cf0d0ce8..e7e8395e2f 100644 --- a/server/src/services/storage.service.ts +++ b/server/src/services/storage.service.ts @@ -45,6 +45,7 @@ export class StorageService extends BaseService { @OnEvent({ name: 'AppBootstrap', priority: BootstrapEventPriority.StorageService }) async onBootstrap() { StorageCore.setMediaLocation(this.detectMediaLocation()); + this.initializeThumbnailStorage(); await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => { const flags = @@ -133,6 +134,12 @@ export class StorageService extends BaseService { }); } + private initializeThumbnailStorage(): void { + const { thumbnailStorage } = this.configRepository.getEnv(); + const path = thumbnailStorage.sqlitePath ?? StorageCore.getThumbnailStoragePath(); + this.thumbnailStorageRepository.initialize(path); + } + @OnJob({ name: JobName.FileDelete, queue: QueueName.BackgroundTask }) async handleDeleteFiles(job: JobOf): Promise { const { files } = job; diff --git a/server/src/utils/file.ts b/server/src/utils/file.ts index 04f1ce48d9..99f06c894a 100644 --- a/server/src/utils/file.ts +++ b/server/src/utils/file.ts @@ -30,6 +30,17 @@ export class ImmichFileResponse { Object.assign(this, response); } } + +export class ImmichBufferResponse { + public readonly data!: Buffer; + public readonly contentType!: string; + public readonly cacheControl!: CacheControl; + public readonly fileName?: string; + + constructor(response: ImmichBufferResponse) { + Object.assign(this, response); + } +} type SendFile = Parameters; type SendFileOptions = SendFile[1]; @@ -82,6 +93,40 @@ export const sendFile = async ( } }; +export const sendBuffer = async ( + res: Response, + next: NextFunction, + handler: () => Promise | ImmichBufferResponse, + logger: LoggingRepository, +): Promise => { + try { + const file = await handler(); + const cacheControlHeader = cacheControlHeaders[file.cacheControl]; + if (cacheControlHeader) { + res.set('Cache-Control', cacheControlHeader); + } + + res.header('Content-Type', file.contentType); + res.header('Content-Length', file.data.length.toString()); + if (file.fileName) { + res.header('Content-Disposition', `inline; filename*=UTF-8''${encodeURIComponent(file.fileName)}`); + } + + res.send(file.data); + } catch (error: Error | any) { + if (isConnectionAborted(error) || res.headersSent) { + return; + } + + if (error instanceof HttpException === false) { + logger.error(`Unable to send buffer: ${error}`, error.stack); + } + + res.header('Cache-Control', 'none'); + next(error); + } +}; + export const asStreamableFile = ({ stream, type, length }: ImmichReadStream) => { return new StreamableFile(stream, { type, length }); }; diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts index 99c08c0fa7..cfcbecf4c5 100644 --- a/server/src/workers/api.ts +++ b/server/src/workers/api.ts @@ -12,6 +12,7 @@ async function bootstrap() { configureTelemetry(); const app = await NestFactory.create(ApiModule, { bufferLogs: true }); + app.enableShutdownHooks(); app.get(AppRepository).setCloseFn(() => app.close()); void configureExpress(app, { diff --git a/server/src/workers/maintenance.ts b/server/src/workers/maintenance.ts index 035ec600af..cfd1961645 100644 --- a/server/src/workers/maintenance.ts +++ b/server/src/workers/maintenance.ts @@ -11,6 +11,7 @@ async function bootstrap() { configureTelemetry(); const app = await NestFactory.create(MaintenanceModule, { bufferLogs: true }); + app.enableShutdownHooks(); app.get(AppRepository).setCloseFn(() => app.close()); void configureExpress(app, { diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts index 8f06b4b0b1..3b471d6a86 100644 --- a/server/src/workers/microservices.ts +++ b/server/src/workers/microservices.ts @@ -1,5 +1,5 @@ import { NestFactory } from '@nestjs/core'; -import { isMainThread } from 'node:worker_threads'; +import { isMainThread, parentPort } from 'node:worker_threads'; import { MicroservicesModule } from 'src/app.module'; import { serverVersion } from 'src/constants'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; @@ -16,6 +16,7 @@ export async function bootstrap() { } const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true }); + app.enableShutdownHooks(); const logger = await app.resolve(LoggingRepository); const configRepository = app.get(ConfigRepository); app.get(AppRepository).setCloseFn(() => app.close()); @@ -29,13 +30,24 @@ export async function bootstrap() { await (host ? app.listen(0, host) : app.listen(0)); logger.log(`Immich Microservices is running [v${serverVersion}] [${environment}] `); + + return app; } if (!isMainThread) { - bootstrap().catch((error) => { - if (!isStartUpError(error)) { - console.error(error); - } - throw error; - }); + bootstrap() + .then((app) => { + parentPort?.on('message', (message) => { + if (message?.type === 'shutdown') { + console.log('Microservices worker received shutdown message'); + void app.close(); + } + }); + }) + .catch((error) => { + if (!isStartUpError(error)) { + console.error(error); + } + throw error; + }); } diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 62e498372e..7f00e203cd 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -84,6 +84,10 @@ const envData: EnvData = { ignoreMountCheckErrors: false, }, + thumbnailStorage: { + sqlitePath: undefined, + }, + telemetry: { apiPort: 8081, microservicesPort: 8082, diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.ts index b6b1e82b52..efa16b4791 100644 --- a/server/test/repositories/media.repository.mock.ts +++ b/server/test/repositories/media.repository.mock.ts @@ -5,6 +5,7 @@ import { Mocked, vitest } from 'vitest'; export const newMediaRepositoryMock = (): Mocked> => { return { generateThumbnail: vitest.fn().mockImplementation(() => Promise.resolve()), + generateThumbnailToBuffer: vitest.fn().mockResolvedValue(Buffer.from('')), writeExif: vitest.fn().mockImplementation(() => Promise.resolve()), copyTagGroup: vitest.fn().mockImplementation(() => Promise.resolve()), generateThumbhash: vitest.fn().mockResolvedValue(Buffer.from('')), diff --git a/server/test/repositories/thumbnail-storage.repository.mock.ts b/server/test/repositories/thumbnail-storage.repository.mock.ts new file mode 100644 index 0000000000..349026e7c4 --- /dev/null +++ b/server/test/repositories/thumbnail-storage.repository.mock.ts @@ -0,0 +1,15 @@ +import { ThumbnailStorageRepository } from 'src/repositories/thumbnail-storage.repository'; +import { RepositoryInterface } from 'src/types'; +import { Mocked, vitest } from 'vitest'; + +export const newThumbnailStorageRepositoryMock = (): Mocked> => { + return { + initialize: vitest.fn(), + isEnabled: vitest.fn().mockReturnValue(false), + store: vitest.fn(), + get: vitest.fn(), + delete: vitest.fn(), + deleteByAsset: vitest.fn(), + close: vitest.fn(), + }; +}; diff --git a/server/test/utils.ts b/server/test/utils.ts index cd866994eb..7a322c4c9f 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -60,6 +60,7 @@ import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; +import { ThumbnailStorageRepository } from 'src/repositories/thumbnail-storage.repository'; import { TrashRepository } from 'src/repositories/trash.repository'; import { UserRepository } from 'src/repositories/user.repository'; import { VersionHistoryRepository } from 'src/repositories/version-history.repository'; @@ -81,6 +82,7 @@ import { newMediaRepositoryMock } from 'test/repositories/media.repository.mock' import { newMetadataRepositoryMock } from 'test/repositories/metadata.repository.mock'; import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock'; import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; +import { newThumbnailStorageRepositoryMock } from 'test/repositories/thumbnail-storage.repository.mock'; import { ITelemetryRepositoryMock, newTelemetryRepositoryMock } from 'test/repositories/telemetry.repository.mock'; import { assert, Mock, Mocked, vitest } from 'vitest'; @@ -255,6 +257,7 @@ export type ServiceOverrides = { systemMetadata: SystemMetadataRepository; tag: TagRepository; telemetry: TelemetryRepository; + thumbnailStorage: ThumbnailStorageRepository; trash: TrashRepository; user: UserRepository; versionHistory: VersionHistoryRepository; @@ -332,6 +335,7 @@ export const getMocks = () => { // eslint-disable-next-line no-sparse-arrays tag: automock(TagRepository, { args: [, loggerMock], strict: false }), telemetry: newTelemetryRepositoryMock(), + thumbnailStorage: newThumbnailStorageRepositoryMock(), trash: automock(TrashRepository), user: automock(UserRepository, { strict: false }), versionHistory: automock(VersionHistoryRepository), @@ -397,6 +401,7 @@ export const newTestService = ( overrides.systemMetadata || (mocks.systemMetadata as As), overrides.tag || (mocks.tag as As), overrides.telemetry || (mocks.telemetry as unknown as TelemetryRepository), + overrides.thumbnailStorage || (mocks.thumbnailStorage as As), overrides.trash || (mocks.trash as As), overrides.user || (mocks.user as As), overrides.versionHistory || (mocks.versionHistory as As),