feat: add checksum algorithm field

server/src/database.ts

@@ -5,6 +5,7 @@ import {
   AssetFileType,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   MemoryType,
   Permission,
   PluginContext,
@@ -111,6 +112,7 @@ export type Memory = {
 export type Asset = {
   id: string;
   checksum: Buffer<ArrayBufferLike>;
+  checksumAlgorithm: ChecksumAlgorithm;
   deviceAssetId: string;
   deviceId: string;
   fileCreatedAt: Date;
@@ -329,6 +331,7 @@ export const columns = {
   asset: [
     'asset.id',
     'asset.checksum',
+    'asset.checksumAlgorithm',
     'asset.deviceAssetId',
     'asset.deviceId',
     'asset.fileCreatedAt',

server/src/dtos/asset-response.dto.ts

@@ -13,7 +13,7 @@ import {
 } from 'src/dtos/person.dto';
 import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';
 import { UserResponseDto, mapUser } from 'src/dtos/user.dto';
-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
 import { ImageDimensions } from 'src/types';
 import { getDimensions } from 'src/utils/asset.util';
 import { hexOrBufferToBase64 } from 'src/utils/bytes';
@@ -147,6 +147,7 @@ export type MapAsset = {
   updateId: string;
   status: AssetStatus;
   checksum: Buffer<ArrayBufferLike>;
+  checksumAlgorithm: ChecksumAlgorithm;
   deviceAssetId: string;
   deviceId: string;
   duplicateId: string | null;

server/src/enum.ts

@@ -37,6 +37,11 @@ export enum AssetType {
   Other = 'OTHER',
 }
 
+export enum ChecksumAlgorithm {
+  sha1File = 'sha1-file', // sha1 checksum of the whole file contents
+  sha1Path = 'sha1-path', // sha1 checksum of "path:" plus the file path, currently used in external libraries, deprecated
+}
+
 export enum AssetFileType {
   /**
    * An full/large-size image extracted/converted from RAW photos

server/src/queries/asset.job.repository.sql

@@ -250,6 +250,7 @@ where
 select
   "asset"."id",
   "asset"."checksum",
+  "asset"."checksumAlgorithm",
   "asset"."deviceAssetId",
   "asset"."deviceId",
   "asset"."fileCreatedAt",

server/src/schema/enums.ts

@@ -1,5 +1,5 @@
 import { registerEnum } from '@immich/sql-tools';
-import { AssetStatus, AssetVisibility, SourceType } from 'src/enum';
+import { AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType } from 'src/enum';
 
 export const assets_status_enum = registerEnum({
   name: 'assets_status_enum',
@@ -15,3 +15,8 @@ export const asset_visibility_enum = registerEnum({
   name: 'asset_visibility_enum',
   values: Object.values(AssetVisibility),
 });
+
+export const asset_checksum_algorithm_enum = registerEnum({
+  name: 'asset_checksum_algorithm_enum',
+  values: Object.values(ChecksumAlgorithm),
+});

server/src/schema/migrations/1772202147901-AddChecksumAlgorithm.ts.ts

@@ -0,0 +1,35 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TYPE "asset_checksum_algorithm_enum" AS ENUM ('sha1-file','sha1-path');`.execute(db);
+  await sql`ALTER TABLE "asset" ADD "checksumAlgorithm" asset_checksum_algorithm_enum;`.execute(db);
+
+  // Update in batches to handle millions of rows efficiently
+  const batchSize = 10_000;
+  let updatedRows: number;
+  
+  do {
+    const result = await sql`
+      UPDATE "asset"
+      SET "checksumAlgorithm" = CASE 
+        WHEN "isExternal" = true THEN 'sha1-path'::asset_checksum_algorithm_enum 
+        ELSE 'sha1-file'::asset_checksum_algorithm_enum 
+      END
+      WHERE "id" IN (
+        SELECT "id" 
+        FROM "asset" 
+        WHERE "checksumAlgorithm" IS NULL 
+        LIMIT ${batchSize}
+      )
+    `.execute(db);
+    
+    updatedRows = Number(result.numAffectedRows ?? 0);
+  } while (updatedRows > 0);
+
+  await sql`ALTER TABLE "asset" ALTER COLUMN "checksumAlgorithm" SET NOT NULL;`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset" DROP COLUMN "checksumAlgorithm";`.execute(db);
+  await sql`DROP TYPE "asset_checksum_algorithm_enum";`.execute(db);
+}

server/src/schema/tables/asset.table.ts

@@ -12,8 +12,8 @@ import {
   UpdateDateColumn,
 } from '@immich/sql-tools';
 import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';
-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
-import { asset_visibility_enum, assets_status_enum } from 'src/schema/enums';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
+import { asset_checksum_algorithm_enum, asset_visibility_enum, assets_status_enum } from 'src/schema/enums';
 import { asset_delete_audit } from 'src/schema/functions';
 import { LibraryTable } from 'src/schema/tables/library.table';
 import { StackTable } from 'src/schema/tables/stack.table';
@@ -98,6 +98,9 @@ export class AssetTable {
   @Column({ type: 'bytea', index: true })
   checksum!: Buffer; // sha1 checksum
 
+  @Column({ enum: asset_checksum_algorithm_enum })
+  checksumAlgorithm!: ChecksumAlgorithm;
+
   @ForeignKeyColumn(() => AssetTable, { nullable: true, onUpdate: 'CASCADE', onDelete: 'SET NULL' })
   livePhotoVideoId!: string | null;
 

server/src/services/asset-media.service.ts

@@ -27,6 +27,7 @@ import {
   AssetStatus,
   AssetVisibility,
   CacheControl,
+  ChecksumAlgorithm,
   JobName,
   Permission,
   StorageFolder,
@@ -409,6 +410,7 @@ export class AssetMediaService extends BaseService {
       deviceId: asset.deviceId,
       type: asset.type,
       checksum: asset.checksum,
+      checksumAlgorithm: asset.checksumAlgorithm,
       fileCreatedAt: asset.fileCreatedAt,
       localDateTime: asset.localDateTime,
       fileModifiedAt: asset.fileModifiedAt,
@@ -430,6 +432,7 @@ export class AssetMediaService extends BaseService {
       libraryId: null,
 
       checksum: file.checksum,
+      checksumAlgorithm: ChecksumAlgorithm.sha1File,
       originalPath: file.originalPath,
 
       deviceAssetId: dto.deviceAssetId,

server/src/services/library.service.ts

@@ -17,7 +17,17 @@ import {
   ValidateLibraryImportPathResponseDto,
   ValidateLibraryResponseDto,
 } from 'src/dtos/library.dto';
-import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
+import {
+  AssetStatus,
+  AssetType,
+  ChecksumAlgorithm,
+  CronJob,
+  DatabaseLock,
+  ImmichWorker,
+  JobName,
+  JobStatus,
+  QueueName,
+} from 'src/enum';
 import { ArgOf } from 'src/repositories/event.repository';
 import { AssetSyncResult } from 'src/repositories/library.repository';
 import { AssetTable } from 'src/schema/tables/asset.table';
@@ -400,6 +410,7 @@ export class LibraryService extends BaseService {
       ownerId,
       libraryId,
       checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`),
+      checksumAlgorithm: ChecksumAlgorithm.sha1Path,
       originalPath: assetPath,
 
       fileCreatedAt: stat.mtime,

server/src/services/metadata.service.spec.ts

@@ -7,6 +7,7 @@ import {
   AssetFileType,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   ExifOrientation,
   ImmichWorker,
   JobName,
@@ -651,6 +652,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
       expect(mocks.asset.create).toHaveBeenCalledWith({
         checksum: expect.any(Buffer),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         deviceAssetId: 'NONE',
         deviceId: 'NONE',
         fileCreatedAt: asset.fileCreatedAt,
@@ -704,6 +706,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
       expect(mocks.asset.create).toHaveBeenCalledWith({
         checksum: expect.any(Buffer),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         deviceAssetId: 'NONE',
         deviceId: 'NONE',
         fileCreatedAt: asset.fileCreatedAt,
@@ -757,6 +760,7 @@ describe(MetadataService.name, () => {
       expect(mocks.storage.readFile).toHaveBeenCalledWith(asset.originalPath, expect.any(Object));
       expect(mocks.asset.create).toHaveBeenCalledWith({
         checksum: expect.any(Buffer),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         deviceAssetId: 'NONE',
         deviceId: 'NONE',
         fileCreatedAt: asset.fileCreatedAt,

server/src/services/metadata.service.ts

@@ -14,6 +14,7 @@ import {
   AssetFileType,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   DatabaseLock,
   ExifOrientation,
   ImmichWorker,
@@ -675,6 +676,7 @@ export class MetadataService extends BaseService {
             fileModifiedAt: stats.mtime,
             localDateTime: dates.localDateTime,
             checksum,
+            checksumAlgorithm: ChecksumAlgorithm.sha1File,
             ownerId: asset.ownerId,
             originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),
             originalFileName: `${parse(asset.originalFileName).name}.mp4`,

server/test/factories/asset.factory.ts

@@ -1,5 +1,5 @@
 import { Selectable } from 'kysely';
-import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetFileType, AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
 import { AssetTable } from 'src/schema/tables/asset.table';
 import { StackTable } from 'src/schema/tables/stack.table';
 import { AssetEditFactory } from 'test/factories/asset-edit.factory';
@@ -51,6 +51,7 @@ export class AssetFactory {
       updateId: newUuidV7(),
       status: AssetStatus.Active,
       checksum: newSha1(),
+      checksumAlgorithm: ChecksumAlgorithm.sha1File,
       deviceAssetId: '',
       deviceId: '',
       duplicateId: null,

server/test/fixtures/shared-link.stub.ts

@@ -1,7 +1,7 @@
 import { UserAdmin } from 'src/database';
 import { MapAsset } from 'src/dtos/asset-response.dto';
 import { SharedLinkResponseDto } from 'src/dtos/shared-link.dto';
-import { AssetStatus, AssetType, AssetVisibility, SharedLinkType } from 'src/enum';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm, SharedLinkType } from 'src/enum';
 import { AssetFactory } from 'test/factories/asset.factory';
 import { authStub } from 'test/fixtures/auth.stub';
 import { userStub } from 'test/fixtures/user.stub';
@@ -94,6 +94,7 @@ export const sharedLinkStub = {
         type: AssetType.Video,
         originalPath: 'fake_path/jpeg',
         checksum: Buffer.from('file hash', 'utf8'),
+        checksumAlgorithm: ChecksumAlgorithm.sha1File,
         fileModifiedAt: today,
         fileCreatedAt: today,
         localDateTime: today,

server/test/medium.factory.ts

@@ -11,6 +11,7 @@ import {
   AlbumUserRole,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   MemoryType,
   SourceType,
   SyncEntityType,
@@ -535,6 +536,7 @@ const assetInsert = (asset: Partial<Insertable<AssetTable>> = {}) => {
     deviceId: '',
     originalFileName: '',
     checksum: randomBytes(32),
+    checksumAlgorithm: ChecksumAlgorithm.sha1File,
     type: AssetType.Image,
     originalPath: '/path/to/something.jpg',
     ownerId: 'not-a-valid-uuid',

server/test/small.factory.ts

@@ -28,6 +28,7 @@ import {
   AssetStatus,
   AssetType,
   AssetVisibility,
+  ChecksumAlgorithm,
   MemoryType,
   Permission,
   SourceType,
@@ -249,6 +250,7 @@ const assetFactory = (
     updateId: newUuidV7(),
     status: AssetStatus.Active,
     checksum: newSha1(),
+    checksumAlgorithm: ChecksumAlgorithm.sha1File,
     deviceAssetId: '',
     deviceId: '',
     duplicateId: null,