Merge remote-tracking branch 'origin' into feat/xxhash

Merge branch 'main' of https://github.com/immich-app/immich into feat/xxhash
feat(server): xxhash
2025-12-06 12:51:32 -08:00 · 2024-10-14 14:20:31 +02:00 · 2024-10-11 23:04:24 +02:00 · 2024-10-11 13:46:13 +02:00
34 changed files with 862 additions and 19 deletions
--- a/e2e/src/api/specs/repair.e2e-spec.ts
+++ b/e2e/src/api/specs/repair.e2e-spec.ts
@@ -0,0 +1,95 @@
+import { LibraryResponseDto, LoginResponseDto, getAllLibraries, scanLibrary } from '@immich/sdk';
+import { readFile } from 'fs/promises';
+import { Socket } from 'socket.io-client';
+import { userDto, uuidDto } from 'src/fixtures';
+import { errorDto } from 'src/responses';
+import { app, asBearerAuth, testAssetDir, testAssetDirInternal, utils } from 'src/utils';
+import request from 'supertest';
+import { utimes } from 'utimes';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { b } from 'vitest/dist/chunks/suite.BMWOKiTe.js';
+
+const scan = async (accessToken: string, id: string) => scanLibrary({ id }, { headers: asBearerAuth(accessToken) });
+
+describe('/repair', () => {
+  let admin: LoginResponseDto;
+  let user: LoginResponseDto;
+  let library: LibraryResponseDto;
+  let websocket: Socket;
+
+  beforeAll(async () => {
+    await utils.resetDatabase();
+    admin = await utils.adminSetup();
+    await utils.resetAdminConfig(admin.accessToken);
+    user = await utils.userSetup(admin.accessToken, userDto.user1);
+    library = await utils.createLibrary(admin.accessToken, { ownerId: admin.userId });
+    websocket = await utils.connectWebsocket(admin.accessToken);
+  });
+
+  afterAll(() => {
+    utils.disconnectWebsocket(websocket);
+    utils.resetTempFolder();
+  });
+
+  beforeEach(() => {
+    utils.resetEvents();
+  });
+
+  describe('POST /check', () => {
+    it('should require authentication', async () => {
+      const { status, body } = await request(app).post('/libraries').send({});
+      expect(status).toBe(401);
+      expect(body).toEqual(errorDto.unauthorized);
+    });
+
+    it('should require admin authentication', async () => {
+      const { status, body } = await request(app)
+        .post('/repair/check')
+        .set('Authorization', `Bearer ${user.accessToken}`)
+        .send({ ownerId: admin.userId });
+
+      expect(status).toBe(403);
+      expect(body).toEqual(errorDto.forbidden);
+    });
+
+    it('should detect a changed original file', async () => {
+      const asset = await utils.createAsset(admin.accessToken, {
+        assetData: {
+          filename: 'polemonium_reptans.jpg',
+          bytes: await readFile(`${testAssetDir}/albums/nature/polemonium_reptans.jpg`),
+        },
+      });
+
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', id: asset.id });
+
+      let assetPath = '';
+      {
+        const { status, body } = await request(app)
+          .get(`/assets/${asset.id}`)
+          .set('Authorization', `Bearer ${admin.accessToken}`);
+        expect(status).toBe(200);
+        assetPath = body.originalPath;
+      }
+
+      utils.flipBitInFile(assetPath, 2, 5);
+
+      const { status, body } = await request(app)
+        .post('/repair/check')
+        .set('Authorization', `Bearer ${admin.accessToken}`)
+        .send({ ownerId: admin.userId });
+
+      expect(status).toBe(201);
+      expect(body).toEqual(
+        expect.arrayContaining([
+        expect.objectContaining({
+          ownerId: admin.userId,
+          name: 'New External Library',
+          refreshedAt: null,
+          assetCount: 0,
+          importPaths: [],
+          exclusionPatterns: expect.any(Array),
+        }),
+      ]);
+    });
+  });
+});
--- a/e2e/src/utils.ts
+++ b/e2e/src/utils.ts
@@ -37,7 +37,7 @@ import {
 import { BrowserContext } from '@playwright/test';
 import { exec, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { dirname } from 'node:path';
 import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
@@ -387,6 +387,20 @@ export const utils = {
    rmSync(path);
  },

+  flipBitInFile: (filePath: string, byteIndex: number, bitPosition: number) => {
+    const data = readFileSync(filePath);
+
+    // Check if the byte index is within the file size
+    if (byteIndex >= data.length) {
+      throw new Error('Byte index is out of range.');
+    }
+
+    // Flip the specific bit using XOR
+    data[byteIndex] ^= 1 << bitPosition;
+
+    writeFileSync(filePath, data);
+  },
+
  removeDirectory: (path: string) => {
    if (!existsSync(path)) {
      return;
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -23,6 +23,7 @@ RUN npm run build
 RUN npm prune --omit=dev --omit=optional
 COPY --from=dev /usr/src/app/node_modules/@img ./node_modules/@img
 COPY --from=dev /usr/src/app/node_modules/exiftool-vendored.pl ./node_modules/exiftool-vendored.pl
+COPY --from=dev /usr/src/app/node_modules/@node-rs ./node_modules/@node-rs

 # web build
 FROM node:20.17.0-alpine3.20@sha256:2d07db07a2df6830718ae2a47db6fedce6745f5bcd174c398f2acdda90a11c03 AS web
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -20,6 +20,7 @@
        "@nestjs/swagger": "^7.1.8",
        "@nestjs/typeorm": "^10.0.0",
        "@nestjs/websockets": "^10.2.2",
+        "@node-rs/xxhash": "^1.7.4",
        "@opentelemetry/auto-instrumentations-node": "^0.50.0",
        "@opentelemetry/context-async-hooks": "^1.24.0",
        "@opentelemetry/exporter-prometheus": "^0.53.0",
@@ -725,6 +726,17 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "node_modules/@emnapi/core": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.3.0.tgz",
+      "integrity": "sha512-9hRqVlhwqBqCoToZ3hFcNVqL+uyHV06Y47ax4UB8L6XgVRqYz7MFnfessojo6+5TK89pKwJnpophwjTMOeKI9Q==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "@emnapi/wasi-threads": "1.0.1",
+        "tslib": "^2.4.0"
+      }
+    },
    "node_modules/@emnapi/runtime": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz",
@@ -734,6 +746,16 @@
        "tslib": "^2.4.0"
      }
    },
+    "node_modules/@emnapi/wasi-threads": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.1.tgz",
+      "integrity": "sha512-iIBu7mwkq4UQGeMEM8bLwNK962nXdhodeScX4slfQnRhEMMzvYivHhutCIk8uojvmASXXPC2WNEjwxFWk72Oqw==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "tslib": "^2.4.0"
+      }
+    },
    "node_modules/@esbuild/aix-ppc64": {
      "version": "0.21.5",
      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
@@ -1969,6 +1991,18 @@
        "darwin"
      ]
    },
+    "node_modules/@napi-rs/wasm-runtime": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.5.tgz",
+      "integrity": "sha512-kwUxR7J9WLutBbulqg1dfOrMTwhMdXLdcGUhcbCcGwnPLt3gz19uHVdwH1syKVDbE022ZS2vZxOWflFLS0YTjw==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "@emnapi/core": "^1.1.0",
+        "@emnapi/runtime": "^1.1.0",
+        "@tybys/wasm-util": "^0.9.0"
+      }
+    },
    "node_modules/@nestjs/bull-shared": {
      "version": "10.2.1",
      "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.2.1.tgz",
@@ -2515,6 +2549,259 @@
        "node": ">= 10"
      }
    },
+    "node_modules/@node-rs/xxhash": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash/-/xxhash-1.7.4.tgz",
+      "integrity": "sha512-NU1YQx1IUlehoHEH2j/SAyVALBAVgI2Btp9//GL816/6Wgd79nz0XxbYG88iFv43T3tRff3i9qE9drHHMTDMFw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 12"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/Brooooooklyn"
+      },
+      "optionalDependencies": {
+        "@node-rs/xxhash-android-arm-eabi": "1.7.4",
+        "@node-rs/xxhash-android-arm64": "1.7.4",
+        "@node-rs/xxhash-darwin-arm64": "1.7.4",
+        "@node-rs/xxhash-darwin-x64": "1.7.4",
+        "@node-rs/xxhash-freebsd-x64": "1.7.4",
+        "@node-rs/xxhash-linux-arm-gnueabihf": "1.7.4",
+        "@node-rs/xxhash-linux-arm64-gnu": "1.7.4",
+        "@node-rs/xxhash-linux-arm64-musl": "1.7.4",
+        "@node-rs/xxhash-linux-x64-gnu": "1.7.4",
+        "@node-rs/xxhash-linux-x64-musl": "1.7.4",
+        "@node-rs/xxhash-wasm32-wasi": "1.7.4",
+        "@node-rs/xxhash-win32-arm64-msvc": "1.7.4",
+        "@node-rs/xxhash-win32-ia32-msvc": "1.7.4",
+        "@node-rs/xxhash-win32-x64-msvc": "1.7.4"
+      }
+    },
+    "node_modules/@node-rs/xxhash-android-arm-eabi": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm-eabi/-/xxhash-android-arm-eabi-1.7.4.tgz",
+      "integrity": "sha512-+Zhpx5X3taDeEPUA20gkWDm2JAQusYyZNVhZLXr+rIgsSUhA8pgc5kJ1jn7NlKnn++ilCrJhHW9T8DRqVDRuZA==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-android-arm64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm64/-/xxhash-android-arm64-1.7.4.tgz",
+      "integrity": "sha512-BZjCr0xfbzX4S1XsM3pxC+PFXKGo05YiQ74AE7+YnXNCJNvIBhDZ7wytYj/Lnx5/azjxKtWFlZzmTV7GwNZBbQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-darwin-arm64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-arm64/-/xxhash-darwin-arm64-1.7.4.tgz",
+      "integrity": "sha512-v4KXiKLpHNe86mTlR/6ManCuDabCZ2c8NfkbXuFaXzqUxOt6eU34NFQqpIGo9+P6M+ncrwaEBS1EtNR0DcPQrA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-darwin-x64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-x64/-/xxhash-darwin-x64-1.7.4.tgz",
+      "integrity": "sha512-E8k4cJSo2lpbloGRU3g4MocezxNMXSGSrQa1yGiTH59Zb1TDZArWlsFIALD8pVyQnjn1JNtINLyjN11Ym92YFg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-freebsd-x64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-freebsd-x64/-/xxhash-freebsd-x64-1.7.4.tgz",
+      "integrity": "sha512-xlqDNdofr7fKFwMrOtIcDWhNMgnCZZ4/mnrxxMuzK0YU5onEp253wYOCdcENzFpnXednpJrsmVhtElBBh9buiw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-linux-arm-gnueabihf": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm-gnueabihf/-/xxhash-linux-arm-gnueabihf-1.7.4.tgz",
+      "integrity": "sha512-4ZM/257gBX8dOICXlCj8xig/Oc22KOZTbmg24qOGeBXLCPLgHrT83XuSJRnCo73nJ66LdhdiKpnS+9ZDZP5XVQ==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-linux-arm64-gnu": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-gnu/-/xxhash-linux-arm64-gnu-1.7.4.tgz",
+      "integrity": "sha512-3YkhVyGWuIuOOzgr517uEHUfDFIbVRg2iBLVi/qTyyGdZlvziZAkiMU6FY7bXu3eTLpfUXHb9R53TNGkDDBM1Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-linux-arm64-musl": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-musl/-/xxhash-linux-arm64-musl-1.7.4.tgz",
+      "integrity": "sha512-0ULea3A/77VLQhBR70REhfYx9ghGl9LZrVeba2/ANzIL9j6F55tjN8c4nN8G/wJNp8kuw8FrJL7V7tOsbZ/tjQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-linux-x64-gnu": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-gnu/-/xxhash-linux-x64-gnu-1.7.4.tgz",
+      "integrity": "sha512-6zrEQQAM30HYku/ET2xLoI1L4q6X1Si9wZtsmc3ZPPmrPCPXksqYKcXJe41M5End+HhBPSU7iIvnBXny8TTmrg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-linux-x64-musl": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-musl/-/xxhash-linux-x64-musl-1.7.4.tgz",
+      "integrity": "sha512-Wa+BzU4uD262NvxQL5mdqmLNWY1yzkKDHItjsA79G73357MNCg6QP+TGhYwTiPYlN1eQa9h0APYNDbBRBXpowg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-wasm32-wasi": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-wasm32-wasi/-/xxhash-wasm32-wasi-1.7.4.tgz",
+      "integrity": "sha512-aQdGSlSGkkBsSXpTnGT97CrmHjBrzagTcp39jAzKpGiTLkZalxHrK3KA3SSFot6OSOBhQLqUAfXvXkEcqPWhsg==",
+      "cpu": [
+        "wasm32"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "@napi-rs/wasm-runtime": "^0.2.3"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@node-rs/xxhash-win32-arm64-msvc": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-arm64-msvc/-/xxhash-win32-arm64-msvc-1.7.4.tgz",
+      "integrity": "sha512-15rssDOpcJPMqUKgb5BmEIQJSohYcX9aWaBVl+RjrRs9m+Pn/gxdWB+HgWy/pfq6zbXhtlnZ/s3BlRjKBKE/EA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-win32-ia32-msvc": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-ia32-msvc/-/xxhash-win32-ia32-msvc-1.7.4.tgz",
+      "integrity": "sha512-gX4hvPeIVxrEluqNCUnYh0h/3vC/VEpBxAYUo8mI33wSt6u6qiDssTe5P/1/06KCOpmASSEkSdmBaaEyJZOaSg==",
+      "cpu": [
+        "ia32"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/@node-rs/xxhash-win32-x64-msvc": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-x64-msvc/-/xxhash-win32-x64-msvc-1.7.4.tgz",
+      "integrity": "sha512-qoOPEJBOq8FJiCWxU1werr4a5efG+tl6HB4L5KXSMhk28ayz0Wjkd3Ld03S4/24WcVia6Q7k4ZeRph0XBcVzVg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 12"
+      }
+    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -5154,6 +5441,16 @@
        "url": "https://opencollective.com/turf"
      }
    },
+    "node_modules/@tybys/wasm-util": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.9.0.tgz",
+      "integrity": "sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "tslib": "^2.4.0"
+      }
+    },
    "node_modules/@types/archiver": {
      "version": "6.0.2",
      "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz",
@@ -15805,6 +16102,16 @@
        }
      }
    },
+    "@emnapi/core": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.3.0.tgz",
+      "integrity": "sha512-9hRqVlhwqBqCoToZ3hFcNVqL+uyHV06Y47ax4UB8L6XgVRqYz7MFnfessojo6+5TK89pKwJnpophwjTMOeKI9Q==",
+      "optional": true,
+      "requires": {
+        "@emnapi/wasi-threads": "1.0.1",
+        "tslib": "^2.4.0"
+      }
+    },
    "@emnapi/runtime": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz",
@@ -15814,6 +16121,15 @@
        "tslib": "^2.4.0"
      }
    },
+    "@emnapi/wasi-threads": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.1.tgz",
+      "integrity": "sha512-iIBu7mwkq4UQGeMEM8bLwNK962nXdhodeScX4slfQnRhEMMzvYivHhutCIk8uojvmASXXPC2WNEjwxFWk72Oqw==",
+      "optional": true,
+      "requires": {
+        "tslib": "^2.4.0"
+      }
+    },
    "@esbuild/aix-ppc64": {
      "version": "0.21.5",
      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
@@ -16489,6 +16805,17 @@
      "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==",
      "optional": true
    },
+    "@napi-rs/wasm-runtime": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.5.tgz",
+      "integrity": "sha512-kwUxR7J9WLutBbulqg1dfOrMTwhMdXLdcGUhcbCcGwnPLt3gz19uHVdwH1syKVDbE022ZS2vZxOWflFLS0YTjw==",
+      "optional": true,
+      "requires": {
+        "@emnapi/core": "^1.1.0",
+        "@emnapi/runtime": "^1.1.0",
+        "@tybys/wasm-util": "^0.9.0"
+      }
+    },
    "@nestjs/bull-shared": {
      "version": "10.2.1",
      "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.2.1.tgz",
@@ -16788,6 +17115,114 @@
      "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==",
      "optional": true
    },
+    "@node-rs/xxhash": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash/-/xxhash-1.7.4.tgz",
+      "integrity": "sha512-NU1YQx1IUlehoHEH2j/SAyVALBAVgI2Btp9//GL816/6Wgd79nz0XxbYG88iFv43T3tRff3i9qE9drHHMTDMFw==",
+      "requires": {
+        "@node-rs/xxhash-android-arm-eabi": "1.7.4",
+        "@node-rs/xxhash-android-arm64": "1.7.4",
+        "@node-rs/xxhash-darwin-arm64": "1.7.4",
+        "@node-rs/xxhash-darwin-x64": "1.7.4",
+        "@node-rs/xxhash-freebsd-x64": "1.7.4",
+        "@node-rs/xxhash-linux-arm-gnueabihf": "1.7.4",
+        "@node-rs/xxhash-linux-arm64-gnu": "1.7.4",
+        "@node-rs/xxhash-linux-arm64-musl": "1.7.4",
+        "@node-rs/xxhash-linux-x64-gnu": "1.7.4",
+        "@node-rs/xxhash-linux-x64-musl": "1.7.4",
+        "@node-rs/xxhash-wasm32-wasi": "1.7.4",
+        "@node-rs/xxhash-win32-arm64-msvc": "1.7.4",
+        "@node-rs/xxhash-win32-ia32-msvc": "1.7.4",
+        "@node-rs/xxhash-win32-x64-msvc": "1.7.4"
+      }
+    },
+    "@node-rs/xxhash-android-arm-eabi": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm-eabi/-/xxhash-android-arm-eabi-1.7.4.tgz",
+      "integrity": "sha512-+Zhpx5X3taDeEPUA20gkWDm2JAQusYyZNVhZLXr+rIgsSUhA8pgc5kJ1jn7NlKnn++ilCrJhHW9T8DRqVDRuZA==",
+      "optional": true
+    },
+    "@node-rs/xxhash-android-arm64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-android-arm64/-/xxhash-android-arm64-1.7.4.tgz",
+      "integrity": "sha512-BZjCr0xfbzX4S1XsM3pxC+PFXKGo05YiQ74AE7+YnXNCJNvIBhDZ7wytYj/Lnx5/azjxKtWFlZzmTV7GwNZBbQ==",
+      "optional": true
+    },
+    "@node-rs/xxhash-darwin-arm64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-arm64/-/xxhash-darwin-arm64-1.7.4.tgz",
+      "integrity": "sha512-v4KXiKLpHNe86mTlR/6ManCuDabCZ2c8NfkbXuFaXzqUxOt6eU34NFQqpIGo9+P6M+ncrwaEBS1EtNR0DcPQrA==",
+      "optional": true
+    },
+    "@node-rs/xxhash-darwin-x64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-darwin-x64/-/xxhash-darwin-x64-1.7.4.tgz",
+      "integrity": "sha512-E8k4cJSo2lpbloGRU3g4MocezxNMXSGSrQa1yGiTH59Zb1TDZArWlsFIALD8pVyQnjn1JNtINLyjN11Ym92YFg==",
+      "optional": true
+    },
+    "@node-rs/xxhash-freebsd-x64": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-freebsd-x64/-/xxhash-freebsd-x64-1.7.4.tgz",
+      "integrity": "sha512-xlqDNdofr7fKFwMrOtIcDWhNMgnCZZ4/mnrxxMuzK0YU5onEp253wYOCdcENzFpnXednpJrsmVhtElBBh9buiw==",
+      "optional": true
+    },
+    "@node-rs/xxhash-linux-arm-gnueabihf": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm-gnueabihf/-/xxhash-linux-arm-gnueabihf-1.7.4.tgz",
+      "integrity": "sha512-4ZM/257gBX8dOICXlCj8xig/Oc22KOZTbmg24qOGeBXLCPLgHrT83XuSJRnCo73nJ66LdhdiKpnS+9ZDZP5XVQ==",
+      "optional": true
+    },
+    "@node-rs/xxhash-linux-arm64-gnu": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-gnu/-/xxhash-linux-arm64-gnu-1.7.4.tgz",
+      "integrity": "sha512-3YkhVyGWuIuOOzgr517uEHUfDFIbVRg2iBLVi/qTyyGdZlvziZAkiMU6FY7bXu3eTLpfUXHb9R53TNGkDDBM1Q==",
+      "optional": true
+    },
+    "@node-rs/xxhash-linux-arm64-musl": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-arm64-musl/-/xxhash-linux-arm64-musl-1.7.4.tgz",
+      "integrity": "sha512-0ULea3A/77VLQhBR70REhfYx9ghGl9LZrVeba2/ANzIL9j6F55tjN8c4nN8G/wJNp8kuw8FrJL7V7tOsbZ/tjQ==",
+      "optional": true
+    },
+    "@node-rs/xxhash-linux-x64-gnu": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-gnu/-/xxhash-linux-x64-gnu-1.7.4.tgz",
+      "integrity": "sha512-6zrEQQAM30HYku/ET2xLoI1L4q6X1Si9wZtsmc3ZPPmrPCPXksqYKcXJe41M5End+HhBPSU7iIvnBXny8TTmrg==",
+      "optional": true
+    },
+    "@node-rs/xxhash-linux-x64-musl": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-linux-x64-musl/-/xxhash-linux-x64-musl-1.7.4.tgz",
+      "integrity": "sha512-Wa+BzU4uD262NvxQL5mdqmLNWY1yzkKDHItjsA79G73357MNCg6QP+TGhYwTiPYlN1eQa9h0APYNDbBRBXpowg==",
+      "optional": true
+    },
+    "@node-rs/xxhash-wasm32-wasi": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-wasm32-wasi/-/xxhash-wasm32-wasi-1.7.4.tgz",
+      "integrity": "sha512-aQdGSlSGkkBsSXpTnGT97CrmHjBrzagTcp39jAzKpGiTLkZalxHrK3KA3SSFot6OSOBhQLqUAfXvXkEcqPWhsg==",
+      "optional": true,
+      "requires": {
+        "@napi-rs/wasm-runtime": "^0.2.3"
+      }
+    },
+    "@node-rs/xxhash-win32-arm64-msvc": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-arm64-msvc/-/xxhash-win32-arm64-msvc-1.7.4.tgz",
+      "integrity": "sha512-15rssDOpcJPMqUKgb5BmEIQJSohYcX9aWaBVl+RjrRs9m+Pn/gxdWB+HgWy/pfq6zbXhtlnZ/s3BlRjKBKE/EA==",
+      "optional": true
+    },
+    "@node-rs/xxhash-win32-ia32-msvc": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-ia32-msvc/-/xxhash-win32-ia32-msvc-1.7.4.tgz",
+      "integrity": "sha512-gX4hvPeIVxrEluqNCUnYh0h/3vC/VEpBxAYUo8mI33wSt6u6qiDssTe5P/1/06KCOpmASSEkSdmBaaEyJZOaSg==",
+      "optional": true
+    },
+    "@node-rs/xxhash-win32-x64-msvc": {
+      "version": "1.7.4",
+      "resolved": "https://registry.npmjs.org/@node-rs/xxhash-win32-x64-msvc/-/xxhash-win32-x64-msvc-1.7.4.tgz",
+      "integrity": "sha512-qoOPEJBOq8FJiCWxU1werr4a5efG+tl6HB4L5KXSMhk28ayz0Wjkd3Ld03S4/24WcVia6Q7k4ZeRph0XBcVzVg==",
+      "optional": true
+    },
    "@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -18485,6 +18920,15 @@
        "tslib": "^2.6.2"
      }
    },
+    "@tybys/wasm-util": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.9.0.tgz",
+      "integrity": "sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==",
+      "optional": true,
+      "requires": {
+        "tslib": "^2.4.0"
+      }
+    },
    "@types/archiver": {
      "version": "6.0.2",
      "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz",
--- a/server/package.json
+++ b/server/package.json
@@ -45,6 +45,7 @@
    "@nestjs/swagger": "^7.1.8",
    "@nestjs/typeorm": "^10.0.0",
    "@nestjs/websockets": "^10.2.2",
+    "@node-rs/xxhash": "^1.7.4",
    "@opentelemetry/auto-instrumentations-node": "^0.50.0",
    "@opentelemetry/context-async-hooks": "^1.24.0",
    "@opentelemetry/exporter-prometheus": "^0.53.0",
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -195,6 +195,7 @@ export const defaults = Object.freeze<SystemConfig>({
    [QueueName.THUMBNAIL_GENERATION]: { concurrency: 3 },
    [QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
    [QueueName.NOTIFICATION]: { concurrency: 5 },
+    [QueueName.REPAIR]: { concurrency: 2 },
  },
  logging: {
    enabled: true,
--- a/server/src/controllers/repair.controller.ts
+++ b/server/src/controllers/repair.controller.ts
@@ -0,0 +1,24 @@
+import { Controller, Get, Post } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { RepairEntity } from 'src/entities/repair.entity';
+import { Auth, Authenticated } from 'src/middleware/auth.guard';
+import { RepairService } from 'src/services/repair.service';
+
+@ApiTags('Repairs')
+@Controller('repairs')
+export class RepairController {
+  constructor(private service: RepairService) {}
+
+  @Get()
+  @Authenticated({ admin: true })
+  getRepairs(@Auth() auth: AuthDto): Promise<RepairEntity[]> {
+    return this.service.getRepairs(auth);
+  }
+
+  @Post('/check')
+  @Authenticated({ admin: true })
+  validateChecksums(@Auth() auth: AuthDto): Promise<RepairEntity[]> {
+    return this.service.getRepairs(auth);
+  }
+}
--- a/server/src/cores/storage.core.ts
+++ b/server/src/cores/storage.core.ts
@@ -275,7 +275,10 @@ export class StorageCore {
  private savePath(pathType: PathType, id: string, newPath: string) {
    switch (pathType) {
      case AssetPathType.ORIGINAL: {
-        return this.assetRepository.update({ id, originalPath: newPath });
+        return Promise.all([
+          this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.ORIGINAL, path: newPath }),
+          this.assetRepository.update({ id, originalPath: newPath }),
+        ]);
      }
      case AssetPathType.PREVIEW: {
        return this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.PREVIEW, path: newPath });
--- a/server/src/dtos/job.dto.ts
+++ b/server/src/dtos/job.dto.ts
@@ -97,4 +97,7 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>

  @ApiProperty({ type: JobStatusDto })
  [QueueName.NOTIFICATION]!: JobStatusDto;
+
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.REPAIR]!: JobStatusDto;
 }
--- a/server/src/dtos/system-config.dto.ts
+++ b/server/src/dtos/system-config.dto.ts
@@ -218,6 +218,12 @@ class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto>
  @IsObject()
  @Type(() => JobSettingsDto)
  [QueueName.NOTIFICATION]!: JobSettingsDto;
+
+  @ApiProperty({ type: JobSettingsDto })
+  @ValidateNested()
+  @IsObject()
+  @Type(() => JobSettingsDto)
+  [QueueName.REPAIR]!: JobSettingsDto;
 }

 class SystemConfigLibraryScanDto {
--- a/server/src/entities/asset-files.entity.ts
+++ b/server/src/entities/asset-files.entity.ts
@@ -35,4 +35,8 @@ export class AssetFileEntity {

  @Column()
  path!: string;
+
+  @Column({ type: 'bytea', nullable: true, default: null })
+  @Index()
+  checksum!: Buffer | null;
 }
--- a/server/src/entities/repair.entity.ts
+++ b/server/src/entities/repair.entity.ts
@@ -0,0 +1,21 @@
+import { AssetFileEntity } from 'src/entities/asset-files.entity';
+import { RepairType } from 'src/enum';
+import { Column, CreateDateColumn, Entity, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
+
+@Entity('repair')
+export class RepairEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string;
+
+  @ManyToOne(() => AssetFileEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
+  assetFile!: AssetFileEntity;
+
+  @CreateDateColumn({ type: 'timestamptz' })
+  createdAt!: Date;
+
+  @Column()
+  type!: RepairType;
+
+  @Column()
+  path!: string;
+}
--- a/server/src/enum.ts
+++ b/server/src/enum.ts
@@ -11,10 +11,15 @@ export enum AssetType {
 }

 export enum AssetFileType {
+  ORIGINAL = 'original',
  PREVIEW = 'preview',
  THUMBNAIL = 'thumbnail',
 }

+export enum RepairType {
+  CHECKSUM_MISMATCH = 'checksum-mismatch',
+}
+
 export enum AlbumUserRole {
  EDITOR = 'editor',
  VIEWER = 'viewer',
--- a/server/src/interfaces/asset.interface.ts
+++ b/server/src/interfaces/asset.interface.ts
@@ -1,3 +1,4 @@
+import { AssetFileEntity } from 'src/entities/asset-files.entity';
 import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
 import { ExifEntity } from 'src/entities/exif.entity';
@@ -145,6 +146,7 @@ export interface UpsertFileOptions {
  assetId: string;
  type: AssetFileType;
  path: string;
+  checksum?: Buffer;
 }

 export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
@@ -194,6 +196,8 @@ export interface IAssetRepository {
  getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
  getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
  getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
+  getFileById(assetFileId: string): Promise<AssetFileEntity | null>;
+  removeFile(assetId: string, type: AssetFileType): Promise<void>;
  upsertFile(file: UpsertFileOptions): Promise<void>;
  upsertFiles(files: UpsertFileOptions[]): Promise<void>;
 }
--- a/server/src/interfaces/crypto.interface.ts
+++ b/server/src/interfaces/crypto.interface.ts
@@ -5,6 +5,8 @@ export interface ICryptoRepository {
  randomUUID(): string;
  hashFile(filePath: string | Buffer): Promise<Buffer>;
  hashSha256(data: string): string;
+  xxHash(value: string): Buffer;
+  xxHashFile(filePath: string | Buffer): Promise<Buffer>;
  verifySha256(data: string, encrypted: string, publicKey: string): boolean;
  hashSha1(data: string | Buffer): Buffer;
  hashBcrypt(data: string | Buffer, saltOrRounds: string | number): Promise<string>;
--- a/server/src/interfaces/job.interface.ts
+++ b/server/src/interfaces/job.interface.ts
@@ -15,6 +15,7 @@ export enum QueueName {
  SIDECAR = 'sidecar',
  LIBRARY = 'library',
  NOTIFICATION = 'notifications',
+  REPAIR = 'repair',
 }

 export type ConcurrentQueueName = Exclude<
@@ -111,6 +112,9 @@ export enum JobName {

  // Version check
  VERSION_CHECK = 'version-check',
+
+  // REPAIR
+  REPAIR_VERIFY_CHECKSUM = 'repair-verify-checksum',
 }

 export const JOBS_ASSET_PAGINATION_SIZE = 1000;
@@ -283,7 +287,10 @@ export type JobItem =
  | { name: JobName.NOTIFY_SIGNUP; data: INotifySignupJob }

  // Version check
-  | { name: JobName.VERSION_CHECK; data: IBaseJob };
+  | { name: JobName.VERSION_CHECK; data: IBaseJob }
+
+  // Repairs
+  | { name: JobName.REPAIR_VERIFY_CHECKSUM; data: IEntityJob };

 export enum JobStatus {
  SUCCESS = 'success',
--- a/server/src/interfaces/repair.interface.ts
+++ b/server/src/interfaces/repair.interface.ts
@@ -0,0 +1,9 @@
+import { RepairEntity } from 'src/entities/repair.entity';
+import { Paginated, PaginationOptions } from 'src/utils/pagination';
+
+export const IRepairRepository = 'IRepairRepository';
+
+export interface IRepairRepository {
+  create(repair: Partial<RepairEntity>): Promise<RepairEntity>;
+  getAll(pagination: PaginationOptions): Paginated<RepairEntity>;
+}
--- a/server/src/middleware/file-upload.interceptor.ts
+++ b/server/src/middleware/file-upload.interceptor.ts
@@ -2,6 +2,7 @@ import { CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } fr
 import { PATH_METADATA } from '@nestjs/common/constants';
 import { Reflector } from '@nestjs/core';
 import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
+import { xxh3 } from '@node-rs/xxhash';
 import { NextFunction, RequestHandler } from 'express';
 import multer, { StorageEngine, diskStorage } from 'multer';
 import { createHash, randomUUID } from 'node:crypto';
@@ -33,12 +34,14 @@ export interface ImmichFile extends Express.Multer.File {
  /** sha1 hash of file */
  uuid: string;
  checksum: Buffer;
+  xxhash: Buffer;
 }

 export function mapToUploadFile(file: ImmichFile): UploadFile {
  return {
    uuid: file.uuid,
    checksum: file.checksum,
+    xxhash: file.xxhash,
    originalPath: file.path,
    originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'),
    size: file.size,
@@ -146,14 +149,26 @@ export class FileUploadInterceptor implements NestInterceptor {
      return;
    }

-    const hash = createHash('sha1');
-    file.stream.on('data', (chunk) => hash.update(chunk));
+    this.logger.debug(`Handling asset upload file: ${file.originalname}`);
+    const xxhash = xxh3.Xxh3.withSeed();
+    const sha1hash = createHash('sha1');
+
+    file.stream.on('data', (chunk) => {
+      xxhash.update(chunk);
+      sha1hash.update(chunk);
+    });
+
    this.defaultStorage._handleFile(request, file, (error, info) => {
      if (error) {
-        hash.destroy();
+        sha1hash.destroy();
+        xxhash.reset();
        callback(error);
      } else {
-        callback(null, { ...info, checksum: hash.digest() });
+        callback(null, {
+          ...info,
+          checksum: sha1hash.digest(),
+          xxhash: Buffer.from(xxhash.digest().toString(16), 'utf8'),
+        });
      }
    });
  }
--- a/server/src/migrations/1728632095015-AddAssetFileChecksum.ts
+++ b/server/src/migrations/1728632095015-AddAssetFileChecksum.ts
@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AssetFileChecksum1728632095015 implements MigrationInterface {
+  name = 'AssetFileChecksum1728632095015';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "asset_files" ADD "checksum" bytea`);
+    await queryRunner.query(`CREATE INDEX "IDX_c946066edd16cfa5c25a26aa8e" ON "asset_files" ("checksum")`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_c946066edd16cfa5c25a26aa8e"`);
+    await queryRunner.query(`ALTER TABLE "asset_files" DROP COLUMN "checksum"`);
+  }
+}
--- a/server/src/queries/asset.repository.sql
+++ b/server/src/queries/asset.repository.sql
@@ -64,7 +64,8 @@ SELECT
  "files"."createdAt" AS "files_createdAt",
  "files"."updatedAt" AS "files_updatedAt",
  "files"."type" AS "files_type",
-  "files"."path" AS "files_path"
+  "files"."path" AS "files_path",
+  "files"."checksum" AS "files_checksum"
 FROM
  "assets" "entity"
  LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "entity"."id"
@@ -248,7 +249,8 @@ SELECT
  "AssetEntity__AssetEntity_files"."createdAt" AS "AssetEntity__AssetEntity_files_createdAt",
  "AssetEntity__AssetEntity_files"."updatedAt" AS "AssetEntity__AssetEntity_files_updatedAt",
  "AssetEntity__AssetEntity_files"."type" AS "AssetEntity__AssetEntity_files_type",
-  "AssetEntity__AssetEntity_files"."path" AS "AssetEntity__AssetEntity_files_path"
+  "AssetEntity__AssetEntity_files"."path" AS "AssetEntity__AssetEntity_files_path",
+  "AssetEntity__AssetEntity_files"."checksum" AS "AssetEntity__AssetEntity_files_checksum"
 FROM
  "assets" "AssetEntity"
  LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
@@ -1117,10 +1119,11 @@ INSERT INTO
    "createdAt",
    "updatedAt",
    "type",
-    "path"
+    "path",
+    "checksum"
  )
 VALUES
-  (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3)
+  (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3, DEFAULT)
 ON CONFLICT ("assetId", "type") DO
 UPDATE
 SET
@@ -1131,7 +1134,8 @@ SET
 RETURNING
  "id",
  "createdAt",
-  "updatedAt"
+  "updatedAt",
+  "checksum"

 -- AssetRepository.upsertFiles
 INSERT INTO
@@ -1141,10 +1145,11 @@ INSERT INTO
    "createdAt",
    "updatedAt",
    "type",
-    "path"
+    "path",
+    "checksum"
  )
 VALUES
-  (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3)
+  (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3, DEFAULT)
 ON CONFLICT ("assetId", "type") DO
 UPDATE
 SET
@@ -1155,4 +1160,5 @@ SET
 RETURNING
  "id",
  "createdAt",
-  "updatedAt"
+  "updatedAt",
+  "checksum"
--- a/server/src/repositories/asset.repository.ts
+++ b/server/src/repositories/asset.repository.ts
@@ -767,8 +767,19 @@ export class AssetRepository implements IAssetRepository {
    return builder.getMany();
  }

+  async removeFile(assetId: string, type: AssetFileType): Promise<void> {
+    await this.fileRepository.delete({ assetId, type });
+  }
+
+  async getFileById(assetFileId: string): Promise<AssetFileEntity | null> {
+    return this.fileRepository.findOne({
+      where: { id: assetFileId },
+      withDeleted: true,
+    });
+  }
+
  @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] })
-  async upsertFile(file: { assetId: string; type: AssetFileType; path: string }): Promise<void> {
+  async upsertFile(file: { assetId: string; type: AssetFileType; path: string; checksum?: Buffer }): Promise<void> {
    await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] });
  }

--- a/server/src/repositories/crypto.repository.ts
+++ b/server/src/repositories/crypto.repository.ts
@@ -1,4 +1,5 @@
 import { Injectable } from '@nestjs/common';
+import { xxh3 } from '@node-rs/xxhash';
 import { compareSync, hash } from 'bcrypt';
 import { createHash, createPublicKey, createVerify, randomBytes, randomUUID } from 'node:crypto';
 import { createReadStream } from 'node:fs';
@@ -28,6 +29,20 @@ export class CryptoRepository implements ICryptoRepository {
    return createHash('sha256').update(value).digest('base64');
  }

+  xxHash(value: string) {
+    return Buffer.from(xxh3.Xxh3.withSeed().update(value).digest().toString(16), 'utf8');
+  }
+
+  xxHashFile(filepath: string | Buffer): Promise<Buffer> {
+    return new Promise<Buffer>((resolve, reject) => {
+      const hash = xxh3.Xxh3.withSeed();
+      const stream = createReadStream(filepath);
+      stream.on('error', (error) => reject(error));
+      stream.on('data', (chunk) => hash.update(chunk));
+      stream.on('end', () => resolve(Buffer.from(hash.digest().toString(16), 'utf8')));
+    });
+  }
+
  verifySha256(value: string, encryptedValue: string, publicKey: string) {
    const publicKeyBuffer = Buffer.from(publicKey, 'base64');
    const cryptoPublicKey = createPublicKey({
--- a/server/src/repositories/index.ts
+++ b/server/src/repositories/index.ts
@@ -23,9 +23,11 @@ import { INotificationRepository } from 'src/interfaces/notification.interface';
 import { IOAuthRepository } from 'src/interfaces/oauth.interface';
 import { IPartnerRepository } from 'src/interfaces/partner.interface';
 import { IPersonRepository } from 'src/interfaces/person.interface';
+import { IRepairRepository } from 'src/interfaces/repair.interface';
 import { ISearchRepository } from 'src/interfaces/search.interface';
 import { IServerInfoRepository } from 'src/interfaces/server-info.interface';
 import { ISessionRepository } from 'src/interfaces/session.interface';
+
 import { ISharedLinkRepository } from 'src/interfaces/shared-link.interface';
 import { IStackRepository } from 'src/interfaces/stack.interface';
 import { IStorageRepository } from 'src/interfaces/storage.interface';
@@ -60,6 +62,7 @@ import { NotificationRepository } from 'src/repositories/notification.repository
 import { OAuthRepository } from 'src/repositories/oauth.repository';
 import { PartnerRepository } from 'src/repositories/partner.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
+import { RepairRepository } from 'src/repositories/repair.repository';
 import { SearchRepository } from 'src/repositories/search.repository';
 import { ServerInfoRepository } from 'src/repositories/server-info.repository';
 import { SessionRepository } from 'src/repositories/session.repository';
@@ -109,6 +112,7 @@ export const repositories = [
  { provide: ITagRepository, useClass: TagRepository },
  { provide: ITrashRepository, useClass: TrashRepository },
  { provide: IUserRepository, useClass: UserRepository },
+  { provide: IRepairRepository, useClass: RepairRepository },
  { provide: IVersionHistoryRepository, useClass: VersionHistoryRepository },
  { provide: IViewRepository, useClass: ViewRepository },
 ];
--- a/server/src/repositories/job.repository.ts
+++ b/server/src/repositories/job.repository.ts
@@ -96,6 +96,9 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {

  // Trash
  [JobName.QUEUE_TRASH_EMPTY]: QueueName.BACKGROUND_TASK,
+
+  // Repair
+  [JobName.REPAIR_VERIFY_CHECKSUM]: QueueName.REPAIR,
 };

@Instrumentation()
--- a/server/src/repositories/repair.repository.ts
+++ b/server/src/repositories/repair.repository.ts
@@ -0,0 +1,27 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { RepairEntity } from 'src/entities/repair.entity';
+import { PaginationMode } from 'src/enum';
+import { IRepairRepository } from 'src/interfaces/repair.interface';
+import { Instrumentation } from 'src/utils/instrumentation';
+import { Paginated, paginatedBuilder, PaginationOptions } from 'src/utils/pagination';
+import { Repository } from 'typeorm';
+
+@Instrumentation()
+@Injectable()
+export class RepairRepository implements IRepairRepository {
+  constructor(@InjectRepository(RepairEntity) private repository: Repository<RepairEntity>) {}
+
+  create(repair: Partial<RepairEntity>): Promise<RepairEntity> {
+    return this.repository.save(repair);
+  }
+
+  getAll(pagination: PaginationOptions): Paginated<RepairEntity> {
+    const builder = this.repository.createQueryBuilder('repair');
+    return paginatedBuilder<RepairEntity>(builder, {
+      mode: PaginationMode.SKIP_TAKE,
+      skip: pagination.skip,
+      take: pagination.take,
+    });
+  }
+}
--- a/server/src/services/asset-media.service.spec.ts
+++ b/server/src/services/asset-media.service.spec.ts
@@ -539,6 +539,7 @@ describe(AssetMediaService.name, () => {
            path: '/path/to/preview',
            type: AssetFileType.THUMBNAIL,
            updatedAt: new Date(),
+            checksum: null,
          },
        ],
      });
@@ -559,6 +560,7 @@ describe(AssetMediaService.name, () => {
            path: '/path/to/preview.jpg',
            type: AssetFileType.PREVIEW,
            updatedAt: new Date(),
+            checksum: null,
          },
        ],
      });
--- a/server/src/services/asset-media.service.ts
+++ b/server/src/services/asset-media.service.ts
@@ -21,7 +21,7 @@ import {
 } from 'src/dtos/asset-media.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
 import { ASSET_CHECKSUM_CONSTRAINT, AssetEntity } from 'src/entities/asset.entity';
-import { AssetStatus, AssetType, CacheControl, Permission, StorageFolder } from 'src/enum';
+import { AssetFileType, AssetStatus, AssetType, CacheControl, Permission, StorageFolder } from 'src/enum';
 import { JobName } from 'src/interfaces/job.interface';
 import { BaseService } from 'src/services/base.service';
 import { requireUploadAccess } from 'src/utils/access';
@@ -39,6 +39,7 @@ export interface UploadRequest {
 export interface UploadFile {
  uuid: string;
  checksum: Buffer;
+  xxhash?: Buffer;
  originalPath: string;
  originalName: string;
  size: number;
@@ -334,6 +335,13 @@ export class AssetMediaService extends BaseService {
      sidecarPath: sidecarPath || null,
    });

+    await this.assetRepository.upsertFile({
+      assetId,
+      type: AssetFileType.ORIGINAL,
+      path: file.originalPath,
+      checksum: file.xxhash,
+    });
+
    await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
    await this.assetRepository.upsertExif({ assetId, fileSizeInByte: file.size });
    await this.jobRepository.queue({
@@ -364,6 +372,8 @@ export class AssetMediaService extends BaseService {
      sidecarPath: asset.sidecarPath,
    });

+    // TODO: asset file original
+
    const { size } = await this.storageRepository.stat(created.originalPath);
    await this.assetRepository.upsertExif({ assetId: created.id, fileSizeInByte: size });
    await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: created.id, source: 'copy' } });
@@ -400,6 +410,13 @@ export class AssetMediaService extends BaseService {
      sidecarPath: sidecarFile?.originalPath,
    });

+    await this.assetRepository.upsertFile({
+      assetId: asset.id,
+      type: AssetFileType.ORIGINAL,
+      path: asset.originalPath,
+      checksum: file.xxhash,
+    });
+
    if (sidecarFile) {
      await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt));
    }
--- a/server/src/services/base.service.ts
+++ b/server/src/services/base.service.ts
@@ -26,6 +26,7 @@ import { INotificationRepository } from 'src/interfaces/notification.interface';
 import { IOAuthRepository } from 'src/interfaces/oauth.interface';
 import { IPartnerRepository } from 'src/interfaces/partner.interface';
 import { IPersonRepository } from 'src/interfaces/person.interface';
+import { IRepairRepository } from 'src/interfaces/repair.interface';
 import { ISearchRepository } from 'src/interfaces/search.interface';
 import { IServerInfoRepository } from 'src/interfaces/server-info.interface';
 import { ISessionRepository } from 'src/interfaces/session.interface';
@@ -70,6 +71,7 @@ export class BaseService {
    @Inject(IOAuthRepository) protected oauthRepository: IOAuthRepository,
    @Inject(IPartnerRepository) protected partnerRepository: IPartnerRepository,
    @Inject(IPersonRepository) protected personRepository: IPersonRepository,
+    @Inject(IRepairRepository) protected repairRepository: IRepairRepository,
    @Inject(ISearchRepository) protected searchRepository: ISearchRepository,
    @Inject(IServerInfoRepository) protected serverInfoRepository: IServerInfoRepository,
    @Inject(ISessionRepository) protected sessionRepository: ISessionRepository,
--- a/server/src/services/index.ts
+++ b/server/src/services/index.ts
@@ -20,6 +20,7 @@ import { MicroservicesService } from 'src/services/microservices.service';
 import { NotificationService } from 'src/services/notification.service';
 import { PartnerService } from 'src/services/partner.service';
 import { PersonService } from 'src/services/person.service';
+import { RepairService } from 'src/services/repair.service';
 import { SearchService } from 'src/services/search.service';
 import { ServerService } from 'src/services/server.service';
 import { SessionService } from 'src/services/session.service';
@@ -63,6 +64,7 @@ export const services = [
  PartnerService,
  PersonService,
  SearchService,
+  RepairService,
  ServerService,
  SessionService,
  SharedLinkService,
--- a/server/src/services/library.service.ts
+++ b/server/src/services/library.service.ts
@@ -16,7 +16,7 @@ import {
 } from 'src/dtos/library.dto';
 import { AssetEntity } from 'src/entities/asset.entity';
 import { LibraryEntity } from 'src/entities/library.entity';
-import { AssetType } from 'src/enum';
+import { AssetFileType, AssetType } from 'src/enum';
 import { DatabaseLock } from 'src/interfaces/database.interface';
 import { ArgOf } from 'src/interfaces/event.interface';
 import {
@@ -420,11 +420,12 @@ export class LibraryService extends BaseService {
      localDateTime: mtime,
      type: assetType,
      originalFileName: parse(assetPath).base,
-
      sidecarPath,
      isExternal: true,
    });

+    await this.assetRepository.upsertFile({ assetId: asset.id, type: AssetFileType.ORIGINAL, path: assetPath });
+
    await this.queuePostSyncJobs(asset);

    return JobStatus.SUCCESS;
@@ -483,6 +484,7 @@ export class LibraryService extends BaseService {
      if (!asset.isOffline) {
        this.logger.debug(`${explanation}, removing: ${asset.originalPath}`);
        await this.assetRepository.updateAll([asset.id], { isOffline: true, deletedAt: new Date() });
+        await this.assetRepository.removeFile(asset.id, AssetFileType.ORIGINAL);
      }
    };

@@ -519,6 +521,12 @@ export class LibraryService extends BaseService {
        fileModifiedAt: mtime,
        originalFileName: parse(asset.originalPath).base,
      });
+
+      await this.assetRepository.upsertFile({
+        assetId: asset.id,
+        type: AssetFileType.ORIGINAL,
+        path: asset.originalPath,
+      });
    }

    if (isAssetModified) {
--- a/server/src/services/microservices.service.ts
+++ b/server/src/services/microservices.service.ts
@@ -12,6 +12,7 @@ import { MediaService } from 'src/services/media.service';
 import { MetadataService } from 'src/services/metadata.service';
 import { NotificationService } from 'src/services/notification.service';
 import { PersonService } from 'src/services/person.service';
+import { RepairService } from 'src/services/repair.service';
 import { SessionService } from 'src/services/session.service';
 import { SmartInfoService } from 'src/services/smart-info.service';
 import { StorageTemplateService } from 'src/services/storage-template.service';
@@ -42,6 +43,7 @@ export class MicroservicesService {
    private userService: UserService,
    private duplicateService: DuplicateService,
    private versionService: VersionService,
+    private repairService: RepairService,
  ) {}

  @OnEvent({ name: 'app.bootstrap' })
@@ -99,6 +101,7 @@ export class MicroservicesService {
      [JobName.TAG_CLEANUP]: () => this.tagService.handleTagCleanup(),
      [JobName.VERSION_CHECK]: () => this.versionService.handleVersionCheck(),
      [JobName.QUEUE_TRASH_EMPTY]: () => this.trashService.handleQueueEmptyTrash(),
+      [JobName.REPAIR_VERIFY_CHECKSUM]: (data) => this.repairService.handleVerifyChecksum(data), //Handles a single path on disk //Watcher calls for new files
    });
  }

--- a/server/src/services/repair.service.ts
+++ b/server/src/services/repair.service.ts
@@ -0,0 +1,66 @@
+import { Injectable } from '@nestjs/common';
+import { DateTime } from 'luxon';
+import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { RepairEntity } from 'src/entities/repair.entity';
+import { Permission, RepairType } from 'src/enum';
+import { IEntityJob, JobName, JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface';
+import { BaseService } from 'src/services/base.service';
+import { usePagination } from 'src/utils/pagination';
+
+@Injectable()
+export class RepairService extends BaseService {
+  async handleVerifyChecksum(job: IEntityJob): Promise<JobStatus> {
+    const assetFile = await this.assetRepository.getFileById(job.id);
+    if (!assetFile) {
+      this.logger.error(`Asset file not found for id: ${job.id}`);
+      return JobStatus.FAILED;
+    }
+
+    if (!assetFile.checksum) {
+      this.logger.error(`Asset file has no checksum, cannot verify: ${job.id}`);
+      return JobStatus.FAILED;
+    }
+    const currentChecksum = await this.cryptoRepository.xxHashFile(assetFile.path);
+    if (currentChecksum.equals(assetFile.checksum)) {
+      this.logger.log(`Asset file checksum verified: ${job.id}`);
+    } else {
+      this.logger.error(`Asset file checksum mismatch: ${job.id}`);
+      await this.repairRepository.create({ assetFile, type: RepairType.CHECKSUM_MISMATCH });
+    }
+    return JobStatus.SUCCESS;
+  }
+
+  async queueVerifyChecksums(auth: AuthDto): Promise<void> {
+    const onlineAssets = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.assetRepository.getAll(pagination),
+    );
+
+    for await (const assets of onlineAssets) {
+      const fileIds = assets
+        .map((asset) => asset.files)
+        .flat()
+        .filter((file) => file.checksum)
+        .map((file) => file.id);
+
+      await this.jobRepository.queueAll(
+        fileIds.map((id) => ({
+          name: JobName.REPAIR_VERIFY_CHECKSUM,
+          data: { id },
+        })),
+      );
+    }
+  }
+
+  async getRepairs(auth: AuthDto): Promise<RepairEntity[]> {
+    let repairs: RepairEntity[] = [];
+    const repairPages = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.repairRepository.getAll(pagination),
+    );
+
+    for await (const repairPage of repairPages) {
+      repairs = repairs.concat(repairPage);
+    }
+    return repairs;
+  }
+}
--- a/server/test/fixtures/asset.stub.ts
+++ b/server/test/fixtures/asset.stub.ts
@@ -15,6 +15,7 @@ const previewFile: AssetFileEntity = {
  path: '/uploads/user-id/thumbs/path.jpg',
  createdAt: new Date('2023-02-23T05:06:29.716Z'),
  updatedAt: new Date('2023-02-23T05:06:29.716Z'),
+  checksum: Buffer.from('file hash', 'utf8'),
 };

 const thumbnailFile: AssetFileEntity = {
@@ -24,6 +25,7 @@ const thumbnailFile: AssetFileEntity = {
  path: '/uploads/user-id/webp/path.ext',
  createdAt: new Date('2023-02-23T05:06:29.716Z'),
  updatedAt: new Date('2023-02-23T05:06:29.716Z'),
+  checksum: Buffer.from('file hash', 'utf8'),
 };

 const files: AssetFileEntity[] = [previewFile, thumbnailFile];
--- a/server/test/repositories/crypto.repository.mock.ts
+++ b/server/test/repositories/crypto.repository.mock.ts
@@ -10,6 +10,7 @@ export const newCryptoRepositoryMock = (): Mocked<ICryptoRepository> => {
    hashSha256: vitest.fn().mockImplementation((input) => `${input} (hashed)`),
    verifySha256: vitest.fn().mockImplementation(() => true),
    hashSha1: vitest.fn().mockImplementation((input) => Buffer.from(`${input.toString()} (hashed)`)),
+    xxHash: vitest.fn().mockImplementation((input) => Buffer.from(`${input.toString()} (hashed)`)),
    hashFile: vitest.fn().mockImplementation((input) => `${input} (file-hashed)`),
    newPassword: vitest.fn().mockReturnValue(Buffer.from('random-bytes').toString('base64')),
  };
Author	SHA1	Message	Date
Jonathan Jogenfors	d75ee28b1f	Merge remote-tracking branch 'origin' into feat/xxhash	2024-10-14 14:20:31 +02:00
Jonathan Jogenfors	c4b7073240	Merge branch 'main' of https://github.com/immich-app/immich into feat/xxhash	2024-10-11 23:04:24 +02:00
Jonathan Jogenfors	0dbb0aabc9	feat(server): xxhash	2024-10-11 13:46:13 +02:00