Compare commits

...

3 Commits

Author SHA1 Message Date
shenlong-tanwen
1a0cd87fed paginate cloud id fetch in migrate cloud id 2026-01-25 21:03:17 +05:30
shenlong-tanwen
d0bb2d7837 tests for reconcileHashesFromCloudId 2026-01-25 20:27:31 +05:30
shenlong-tanwen
290cd0eb6b fix: slow hash reconcilation 2026-01-25 20:27:31 +05:30
13 changed files with 9318 additions and 171 deletions

File diff suppressed because one or more lines are too long

View File

@@ -41,7 +41,7 @@ class HashService {
final Stopwatch stopwatch = Stopwatch()..start();
try {
// Migrate hashes from cloud ID to local ID so we don't have to re-hash them
await _migrateHashes();
await _localAssetRepository.reconcileHashesFromCloudId();
// Sorted by backupSelection followed by isCloud
final localAlbums = await _localAlbumRepository.getBackupAlbums();
@@ -78,15 +78,6 @@ class HashService {
_log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
}
Future<void> _migrateHashes() async {
final hashMappings = await _localAssetRepository.getHashMappingFromCloudId();
if (hashMappings.isEmpty) {
return;
}
await _localAssetRepository.updateHashes(hashMappings);
}
/// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB
/// with hash for those that were successfully hashed. Hashes are looked up in a table
/// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.

View File

@@ -50,24 +50,51 @@ Future<void> syncCloudIds(ProviderContainer ref) async {
return;
}
final mappingsToUpdate = await _fetchCloudIdMappings(db, currentUser.id);
// Deduplicate mappings as a single remote asset ID can match multiple local assets
final seenRemoteAssetIds = <String>{};
final uniqueMapping = mappingsToUpdate.where((mapping) {
if (!seenRemoteAssetIds.add(mapping.remoteAssetId)) {
logger.fine('Duplicate remote asset ID found: ${mapping.remoteAssetId}. Skipping duplicate entry.');
return false;
}
return true;
}).toList();
final assetApi = ref.read(apiServiceProvider).assetsApi;
if (canBulkUpdateMetadata) {
await _bulkUpdateCloudIds(assetApi, uniqueMapping);
return;
// Process cloud IDs in paginated batches
await _processCloudIdMappingsInBatches(db, currentUser.id, assetApi, canBulkUpdateMetadata, logger);
}
Future<void> _processCloudIdMappingsInBatches(
Drift drift,
String userId,
AssetsApi assetsApi,
bool canBulkUpdate,
Logger logger,
) async {
const pageSize = 20000;
int offset = 0;
final seenRemoteAssetIds = <String>{};
while (true) {
final mappings = await _fetchCloudIdMappings(drift, userId, pageSize, offset);
if (mappings.isEmpty) {
break;
}
final uniqueMappings = <_CloudIdMapping>[];
for (final mapping in mappings) {
if (seenRemoteAssetIds.add(mapping.remoteAssetId)) {
uniqueMappings.add(mapping);
} else {
logger.fine('Duplicate remote asset ID found: ${mapping.remoteAssetId}. Skipping duplicate entry.');
}
}
if (uniqueMappings.isNotEmpty) {
if (canBulkUpdate) {
await _bulkUpdateCloudIds(assetsApi, uniqueMappings);
} else {
await _sequentialUpdateCloudIds(assetsApi, uniqueMappings);
}
}
offset += pageSize;
if (mappings.length < pageSize) {
break;
}
}
await _sequentialUpdateCloudIds(assetApi, uniqueMapping);
}
Future<void> _sequentialUpdateCloudIds(AssetsApi assetsApi, List<_CloudIdMapping> mappings) async {
@@ -91,31 +118,26 @@ Future<void> _sequentialUpdateCloudIds(AssetsApi assetsApi, List<_CloudIdMapping
}
Future<void> _bulkUpdateCloudIds(AssetsApi assetsApi, List<_CloudIdMapping> mappings) async {
const batchSize = 10000;
for (int i = 0; i < mappings.length; i += batchSize) {
final endIndex = (i + batchSize > mappings.length) ? mappings.length : i + batchSize;
final batch = mappings.sublist(i, endIndex);
final items = <AssetMetadataBulkUpsertItemDto>[];
for (final mapping in batch) {
items.add(
AssetMetadataBulkUpsertItemDto(
assetId: mapping.remoteAssetId,
key: kMobileMetadataKey,
value: RemoteAssetMobileAppMetadata(
cloudId: mapping.localAsset.cloudId,
createdAt: mapping.localAsset.createdAt.toIso8601String(),
adjustmentTime: mapping.localAsset.adjustmentTime?.toIso8601String(),
latitude: mapping.localAsset.latitude?.toString(),
longitude: mapping.localAsset.longitude?.toString(),
),
final items = <AssetMetadataBulkUpsertItemDto>[];
for (final mapping in mappings) {
items.add(
AssetMetadataBulkUpsertItemDto(
assetId: mapping.remoteAssetId,
key: kMobileMetadataKey,
value: RemoteAssetMobileAppMetadata(
cloudId: mapping.localAsset.cloudId,
createdAt: mapping.localAsset.createdAt.toIso8601String(),
adjustmentTime: mapping.localAsset.adjustmentTime?.toIso8601String(),
latitude: mapping.localAsset.latitude?.toString(),
longitude: mapping.localAsset.longitude?.toString(),
),
);
}
try {
await assetsApi.updateBulkAssetMetadata(AssetMetadataBulkUpsertDto(items: items));
} catch (error, stack) {
Logger('migrateCloudIds').warning('Failed to bulk update metadata', error, stack);
}
),
);
}
try {
await assetsApi.updateBulkAssetMetadata(AssetMetadataBulkUpsertDto(items: items));
} catch (error, stack) {
Logger('migrateCloudIds').warning('Failed to bulk update metadata', error, stack);
}
}
@@ -141,31 +163,34 @@ Future<void> _populateCloudIds(Drift drift) async {
typedef _CloudIdMapping = ({String remoteAssetId, LocalAsset localAsset});
Future<List<_CloudIdMapping>> _fetchCloudIdMappings(Drift drift, String userId) async {
Future<List<_CloudIdMapping>> _fetchCloudIdMappings(Drift drift, String userId, int limit, int offset) async {
final query =
drift.remoteAssetEntity.select().join([
leftOuterJoin(
drift.localAssetEntity,
drift.localAssetEntity.checksum.equalsExp(drift.remoteAssetEntity.checksum),
),
leftOuterJoin(
drift.remoteAssetCloudIdEntity,
drift.remoteAssetEntity.id.equalsExp(drift.remoteAssetCloudIdEntity.assetId),
useColumns: false,
),
])..where(
// Only select assets that have a local cloud ID but either no remote cloud ID or a mismatched eTag
drift.localAssetEntity.id.isNotNull() &
drift.localAssetEntity.iCloudId.isNotNull() &
drift.remoteAssetEntity.ownerId.equals(userId) &
// Skip locked assets as we cannot update them without unlocking first
drift.remoteAssetEntity.visibility.isNotValue(AssetVisibility.locked.index) &
(drift.remoteAssetCloudIdEntity.cloudId.isNull() |
drift.remoteAssetCloudIdEntity.adjustmentTime.isNotExp(drift.localAssetEntity.adjustmentTime) |
drift.remoteAssetCloudIdEntity.latitude.isNotExp(drift.localAssetEntity.latitude) |
drift.remoteAssetCloudIdEntity.longitude.isNotExp(drift.localAssetEntity.longitude) |
drift.remoteAssetCloudIdEntity.createdAt.isNotExp(drift.localAssetEntity.createdAt)),
);
drift.localAssetEntity.select().join([
innerJoin(
drift.remoteAssetEntity,
drift.localAssetEntity.checksum.equalsExp(drift.remoteAssetEntity.checksum),
),
leftOuterJoin(
drift.remoteAssetCloudIdEntity,
drift.remoteAssetEntity.id.equalsExp(drift.remoteAssetCloudIdEntity.assetId),
useColumns: false,
),
])
..where(
// Only select assets that have a local cloud ID but either no remote cloud ID or a mismatched eTag
drift.localAssetEntity.iCloudId.isNotNull() &
drift.remoteAssetEntity.ownerId.equals(userId) &
// Skip locked assets as we cannot update them without unlocking first
drift.remoteAssetEntity.visibility.isNotValue(AssetVisibility.locked.index) &
(drift.remoteAssetCloudIdEntity.cloudId.isNull() |
drift.remoteAssetCloudIdEntity.adjustmentTime.isNotExp(drift.localAssetEntity.adjustmentTime) |
drift.remoteAssetCloudIdEntity.latitude.isNotExp(drift.localAssetEntity.latitude) |
drift.remoteAssetCloudIdEntity.longitude.isNotExp(drift.localAssetEntity.longitude) |
drift.remoteAssetCloudIdEntity.createdAt.isNotExp(drift.localAssetEntity.createdAt)),
)
..orderBy([OrderingTerm.asc(drift.localAssetEntity.id)])
..limit(limit, offset: offset);
return query.map((row) {
return (
remoteAssetId: row.read(drift.remoteAssetEntity.id)!,

View File

@@ -2,6 +2,7 @@ import 'package:drift/drift.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.dart';
import 'package:immich_mobile/infrastructure/utils/drift_default.mixin.dart';
@TableIndex.sql('CREATE INDEX IF NOT EXISTS idx_remote_asset_cloud_id ON remote_asset_cloud_id_entity (cloud_id)')
class RemoteAssetCloudIdEntity extends Table with DriftDefaultsMixin {
TextColumn get assetId => text().references(RemoteAssetEntity, #id, onDelete: KeyAction.cascade)();

View File

@@ -403,6 +403,10 @@ typedef $$RemoteAssetCloudIdEntityTableProcessedTableManager =
i1.RemoteAssetCloudIdEntityData,
i0.PrefetchHooks Function({bool assetId})
>;
i0.Index get idxRemoteAssetCloudId => i0.Index(
'idx_remote_asset_cloud_id',
'CREATE INDEX IF NOT EXISTS idx_remote_asset_cloud_id ON remote_asset_cloud_id_entity (cloud_id)',
);
class $RemoteAssetCloudIdEntityTable extends i2.RemoteAssetCloudIdEntity
with

View File

@@ -97,7 +97,7 @@ class Drift extends $Drift implements IDatabaseRepository {
}
@override
int get schemaVersion => 17;
int get schemaVersion => 18;
@override
MigrationStrategy get migration => MigrationStrategy(
@@ -204,6 +204,9 @@ class Drift extends $Drift implements IDatabaseRepository {
from16To17: (m, v17) async {
await m.addColumn(v17.remoteAssetEntity, v17.remoteAssetEntity.isEdited);
},
from17To18: (m, v18) async {
await m.createIndex(v18.idxRemoteAssetCloudId);
},
),
);

View File

@@ -120,6 +120,7 @@ abstract class $Drift extends i0.GeneratedDatabase {
storeEntity,
trashedLocalAssetEntity,
i11.idxLatLng,
i14.idxRemoteAssetCloudId,
i20.idxTrashedLocalAssetChecksum,
i20.idxTrashedLocalAssetAlbum,
];

View File

@@ -7408,6 +7408,455 @@ i1.GeneratedColumn<bool> _column_101(String aliasedName) =>
),
defaultValue: const CustomExpression('0'),
);
final class Schema18 extends i0.VersionedSchema {
Schema18({required super.database}) : super(version: 18);
@override
late final List<i1.DatabaseSchemaEntity> entities = [
userEntity,
remoteAssetEntity,
stackEntity,
localAssetEntity,
remoteAlbumEntity,
localAlbumEntity,
localAlbumAssetEntity,
idxLocalAssetChecksum,
idxLocalAssetCloudId,
idxRemoteAssetOwnerChecksum,
uQRemoteAssetsOwnerChecksum,
uQRemoteAssetsOwnerLibraryChecksum,
idxRemoteAssetChecksum,
authUserEntity,
userMetadataEntity,
partnerEntity,
remoteExifEntity,
remoteAlbumAssetEntity,
remoteAlbumUserEntity,
remoteAssetCloudIdEntity,
memoryEntity,
memoryAssetEntity,
personEntity,
assetFaceEntity,
storeEntity,
trashedLocalAssetEntity,
idxLatLng,
idxRemoteAssetCloudId,
idxTrashedLocalAssetChecksum,
idxTrashedLocalAssetAlbum,
];
late final Shape20 userEntity = Shape20(
source: i0.VersionedTable(
entityName: 'user_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_1,
_column_3,
_column_84,
_column_85,
_column_91,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape28 remoteAssetEntity = Shape28(
source: i0.VersionedTable(
entityName: 'remote_asset_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_1,
_column_8,
_column_9,
_column_5,
_column_10,
_column_11,
_column_12,
_column_0,
_column_13,
_column_14,
_column_15,
_column_16,
_column_17,
_column_18,
_column_19,
_column_20,
_column_21,
_column_86,
_column_101,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape3 stackEntity = Shape3(
source: i0.VersionedTable(
entityName: 'stack_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [_column_0, _column_9, _column_5, _column_15, _column_75],
attachedDatabase: database,
),
alias: null,
);
late final Shape26 localAssetEntity = Shape26(
source: i0.VersionedTable(
entityName: 'local_asset_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_1,
_column_8,
_column_9,
_column_5,
_column_10,
_column_11,
_column_12,
_column_0,
_column_22,
_column_14,
_column_23,
_column_98,
_column_96,
_column_46,
_column_47,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape9 remoteAlbumEntity = Shape9(
source: i0.VersionedTable(
entityName: 'remote_album_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_1,
_column_56,
_column_9,
_column_5,
_column_15,
_column_57,
_column_58,
_column_59,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape19 localAlbumEntity = Shape19(
source: i0.VersionedTable(
entityName: 'local_album_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_1,
_column_5,
_column_31,
_column_32,
_column_90,
_column_33,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape22 localAlbumAssetEntity = Shape22(
source: i0.VersionedTable(
entityName: 'local_album_asset_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(asset_id, album_id)'],
columns: [_column_34, _column_35, _column_33],
attachedDatabase: database,
),
alias: null,
);
final i1.Index idxLocalAssetChecksum = i1.Index(
'idx_local_asset_checksum',
'CREATE INDEX IF NOT EXISTS idx_local_asset_checksum ON local_asset_entity (checksum)',
);
final i1.Index idxLocalAssetCloudId = i1.Index(
'idx_local_asset_cloud_id',
'CREATE INDEX IF NOT EXISTS idx_local_asset_cloud_id ON local_asset_entity (i_cloud_id)',
);
final i1.Index idxRemoteAssetOwnerChecksum = i1.Index(
'idx_remote_asset_owner_checksum',
'CREATE INDEX IF NOT EXISTS idx_remote_asset_owner_checksum ON remote_asset_entity (owner_id, checksum)',
);
final i1.Index uQRemoteAssetsOwnerChecksum = i1.Index(
'UQ_remote_assets_owner_checksum',
'CREATE UNIQUE INDEX IF NOT EXISTS UQ_remote_assets_owner_checksum ON remote_asset_entity (owner_id, checksum) WHERE(library_id IS NULL)',
);
final i1.Index uQRemoteAssetsOwnerLibraryChecksum = i1.Index(
'UQ_remote_assets_owner_library_checksum',
'CREATE UNIQUE INDEX IF NOT EXISTS UQ_remote_assets_owner_library_checksum ON remote_asset_entity (owner_id, library_id, checksum) WHERE(library_id IS NOT NULL)',
);
final i1.Index idxRemoteAssetChecksum = i1.Index(
'idx_remote_asset_checksum',
'CREATE INDEX IF NOT EXISTS idx_remote_asset_checksum ON remote_asset_entity (checksum)',
);
late final Shape21 authUserEntity = Shape21(
source: i0.VersionedTable(
entityName: 'auth_user_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_1,
_column_3,
_column_2,
_column_84,
_column_85,
_column_92,
_column_93,
_column_7,
_column_94,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape4 userMetadataEntity = Shape4(
source: i0.VersionedTable(
entityName: 'user_metadata_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(user_id, "key")'],
columns: [_column_25, _column_26, _column_27],
attachedDatabase: database,
),
alias: null,
);
late final Shape5 partnerEntity = Shape5(
source: i0.VersionedTable(
entityName: 'partner_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(shared_by_id, shared_with_id)'],
columns: [_column_28, _column_29, _column_30],
attachedDatabase: database,
),
alias: null,
);
late final Shape8 remoteExifEntity = Shape8(
source: i0.VersionedTable(
entityName: 'remote_exif_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(asset_id)'],
columns: [
_column_36,
_column_37,
_column_38,
_column_39,
_column_40,
_column_41,
_column_11,
_column_10,
_column_42,
_column_43,
_column_44,
_column_45,
_column_46,
_column_47,
_column_48,
_column_49,
_column_50,
_column_51,
_column_52,
_column_53,
_column_54,
_column_55,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape7 remoteAlbumAssetEntity = Shape7(
source: i0.VersionedTable(
entityName: 'remote_album_asset_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(asset_id, album_id)'],
columns: [_column_36, _column_60],
attachedDatabase: database,
),
alias: null,
);
late final Shape10 remoteAlbumUserEntity = Shape10(
source: i0.VersionedTable(
entityName: 'remote_album_user_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(album_id, user_id)'],
columns: [_column_60, _column_25, _column_61],
attachedDatabase: database,
),
alias: null,
);
late final Shape27 remoteAssetCloudIdEntity = Shape27(
source: i0.VersionedTable(
entityName: 'remote_asset_cloud_id_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(asset_id)'],
columns: [
_column_36,
_column_99,
_column_100,
_column_96,
_column_46,
_column_47,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape11 memoryEntity = Shape11(
source: i0.VersionedTable(
entityName: 'memory_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_9,
_column_5,
_column_18,
_column_15,
_column_8,
_column_62,
_column_63,
_column_64,
_column_65,
_column_66,
_column_67,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape12 memoryAssetEntity = Shape12(
source: i0.VersionedTable(
entityName: 'memory_asset_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(asset_id, memory_id)'],
columns: [_column_36, _column_68],
attachedDatabase: database,
),
alias: null,
);
late final Shape14 personEntity = Shape14(
source: i0.VersionedTable(
entityName: 'person_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_9,
_column_5,
_column_15,
_column_1,
_column_69,
_column_71,
_column_72,
_column_73,
_column_74,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape15 assetFaceEntity = Shape15(
source: i0.VersionedTable(
entityName: 'asset_face_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [
_column_0,
_column_36,
_column_76,
_column_77,
_column_78,
_column_79,
_column_80,
_column_81,
_column_82,
_column_83,
],
attachedDatabase: database,
),
alias: null,
);
late final Shape18 storeEntity = Shape18(
source: i0.VersionedTable(
entityName: 'store_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id)'],
columns: [_column_87, _column_88, _column_89],
attachedDatabase: database,
),
alias: null,
);
late final Shape25 trashedLocalAssetEntity = Shape25(
source: i0.VersionedTable(
entityName: 'trashed_local_asset_entity',
withoutRowId: true,
isStrict: true,
tableConstraints: ['PRIMARY KEY(id, album_id)'],
columns: [
_column_1,
_column_8,
_column_9,
_column_5,
_column_10,
_column_11,
_column_12,
_column_0,
_column_95,
_column_22,
_column_14,
_column_23,
_column_97,
],
attachedDatabase: database,
),
alias: null,
);
final i1.Index idxLatLng = i1.Index(
'idx_lat_lng',
'CREATE INDEX IF NOT EXISTS idx_lat_lng ON remote_exif_entity (latitude, longitude)',
);
final i1.Index idxRemoteAssetCloudId = i1.Index(
'idx_remote_asset_cloud_id',
'CREATE INDEX IF NOT EXISTS idx_remote_asset_cloud_id ON remote_asset_cloud_id_entity (cloud_id)',
);
final i1.Index idxTrashedLocalAssetChecksum = i1.Index(
'idx_trashed_local_asset_checksum',
'CREATE INDEX IF NOT EXISTS idx_trashed_local_asset_checksum ON trashed_local_asset_entity (checksum)',
);
final i1.Index idxTrashedLocalAssetAlbum = i1.Index(
'idx_trashed_local_asset_album',
'CREATE INDEX IF NOT EXISTS idx_trashed_local_asset_album ON trashed_local_asset_entity (album_id)',
);
}
i0.MigrationStepWithVersion migrationSteps({
required Future<void> Function(i1.Migrator m, Schema2 schema) from1To2,
required Future<void> Function(i1.Migrator m, Schema3 schema) from2To3,
@@ -7425,6 +7874,7 @@ i0.MigrationStepWithVersion migrationSteps({
required Future<void> Function(i1.Migrator m, Schema15 schema) from14To15,
required Future<void> Function(i1.Migrator m, Schema16 schema) from15To16,
required Future<void> Function(i1.Migrator m, Schema17 schema) from16To17,
required Future<void> Function(i1.Migrator m, Schema18 schema) from17To18,
}) {
return (currentVersion, database) async {
switch (currentVersion) {
@@ -7508,6 +7958,11 @@ i0.MigrationStepWithVersion migrationSteps({
final migrator = i1.Migrator(database, schema);
await from16To17(migrator, schema);
return 17;
case 17:
final schema = Schema18(database: database);
final migrator = i1.Migrator(database, schema);
await from17To18(migrator, schema);
return 18;
default:
throw ArgumentError.value('Unknown migration from $currentVersion');
}
@@ -7531,6 +7986,7 @@ i1.OnUpgrade stepByStep({
required Future<void> Function(i1.Migrator m, Schema15 schema) from14To15,
required Future<void> Function(i1.Migrator m, Schema16 schema) from15To16,
required Future<void> Function(i1.Migrator m, Schema17 schema) from16To17,
required Future<void> Function(i1.Migrator m, Schema18 schema) from17To18,
}) => i0.VersionedSchema.stepByStepHelper(
step: migrationSteps(
from1To2: from1To2,
@@ -7549,5 +8005,6 @@ i1.OnUpgrade stepByStep({
from14To15: from14To15,
from15To16: from15To16,
from16To17: from16To17,
from17To18: from17To18,
),
);

View File

@@ -204,34 +204,24 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository {
return query.map((row) => row.toDto()).get();
}
Future<Map<String, String>> getHashMappingFromCloudId() async {
final query =
_db.localAssetEntity.selectOnly().join([
leftOuterJoin(
_db.remoteAssetCloudIdEntity,
_db.localAssetEntity.iCloudId.equalsExp(_db.remoteAssetCloudIdEntity.cloudId),
useColumns: false,
),
leftOuterJoin(
_db.remoteAssetEntity,
_db.remoteAssetCloudIdEntity.assetId.equalsExp(_db.remoteAssetEntity.id),
useColumns: false,
),
])
..addColumns([_db.localAssetEntity.id, _db.remoteAssetEntity.checksum])
..where(
_db.remoteAssetCloudIdEntity.cloudId.isNotNull() &
_db.localAssetEntity.checksum.isNull() &
((_db.remoteAssetCloudIdEntity.adjustmentTime.isExp(_db.localAssetEntity.adjustmentTime)) &
(_db.remoteAssetCloudIdEntity.latitude.isExp(_db.localAssetEntity.latitude)) &
(_db.remoteAssetCloudIdEntity.longitude.isExp(_db.localAssetEntity.longitude)) &
(_db.remoteAssetCloudIdEntity.createdAt.isExp(_db.localAssetEntity.createdAt))),
);
final mapping = await query
.map(
(row) => (assetId: row.read(_db.localAssetEntity.id)!, checksum: row.read(_db.remoteAssetEntity.checksum)!),
)
.get();
return {for (final entry in mapping) entry.assetId: entry.checksum};
Future<void> reconcileHashesFromCloudId() async {
await _db.customUpdate(
'''
UPDATE local_asset_entity
SET checksum = remote_asset_entity.checksum
FROM remote_asset_cloud_id_entity
INNER JOIN remote_asset_entity
ON remote_asset_cloud_id_entity.asset_id = remote_asset_entity.id
WHERE local_asset_entity.i_cloud_id = remote_asset_cloud_id_entity.cloud_id
AND local_asset_entity.i_cloud_id IS NOT NULL
AND local_asset_entity.checksum IS NULL
AND remote_asset_cloud_id_entity.adjustment_time IS local_asset_entity.adjustment_time
AND remote_asset_cloud_id_entity.latitude IS local_asset_entity.latitude
AND remote_asset_cloud_id_entity.longitude IS local_asset_entity.longitude
AND remote_asset_cloud_id_entity.created_at IS local_asset_entity.created_at
''',
updates: {_db.localAssetEntity},
updateKind: UpdateKind.update,
);
}
}

View File

@@ -33,7 +33,7 @@ void main() {
registerFallbackValue(LocalAssetStub.image1);
registerFallbackValue(<String, String>{});
when(() => mockAssetRepo.getHashMappingFromCloudId()).thenAnswer((_) async => {});
when(() => mockAssetRepo.reconcileHashesFromCloudId()).thenAnswer((_) async => {});
when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
});
@@ -191,5 +191,4 @@ void main() {
verify(() => mockNativeApi.hashAssets([asset2.id], allowNetworkAccess: false)).called(1);
});
});
}

View File

@@ -20,6 +20,7 @@ import 'schema_v14.dart' as v14;
import 'schema_v15.dart' as v15;
import 'schema_v16.dart' as v16;
import 'schema_v17.dart' as v17;
import 'schema_v18.dart' as v18;
class GeneratedHelper implements SchemaInstantiationHelper {
@override
@@ -59,6 +60,8 @@ class GeneratedHelper implements SchemaInstantiationHelper {
return v16.DatabaseAtV16(db);
case 17:
return v17.DatabaseAtV17(db);
case 18:
return v18.DatabaseAtV18(db);
default:
throw MissingSchemaException(version, versions);
}
@@ -82,5 +85,6 @@ class GeneratedHelper implements SchemaInstantiationHelper {
15,
16,
17,
18,
];
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,4 +1,4 @@
import 'package:drift/drift.dart';
import 'package:drift/drift.dart' hide isNull;
import 'package:drift/native.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/constants/enums.dart';
@@ -8,11 +8,13 @@ import 'package:immich_mobile/infrastructure/entities/local_album.entity.drift.d
import 'package:immich_mobile/infrastructure/entities/local_album_asset.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/remote_asset_cloud_id.entity.drift.dart';
import 'package:immich_mobile/infrastructure/entities/user.entity.drift.dart';
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
void main() {
final now = DateTime(2024, 1, 15);
late Drift db;
late DriftLocalAssetRepository repository;
@@ -25,68 +27,98 @@ void main() {
await db.close();
});
Future<void> insertLocalAsset({
required String id,
String? checksum,
DateTime? createdAt,
AssetType type = AssetType.image,
bool isFavorite = false,
String? iCloudId,
DateTime? adjustmentTime,
double? latitude,
double? longitude,
}) async {
final created = createdAt ?? now;
await db
.into(db.localAssetEntity)
.insert(
LocalAssetEntityCompanion.insert(
id: id,
name: 'asset_$id.jpg',
checksum: Value(checksum),
type: type,
createdAt: Value(created),
updatedAt: Value(created),
isFavorite: Value(isFavorite),
iCloudId: Value(iCloudId),
adjustmentTime: Value(adjustmentTime),
latitude: Value(latitude),
longitude: Value(longitude),
),
);
}
Future<void> insertRemoteAsset({
required String id,
required String checksum,
required String ownerId,
DateTime? deletedAt,
}) async {
await db
.into(db.remoteAssetEntity)
.insert(
RemoteAssetEntityCompanion.insert(
id: id,
name: 'remote_$id.jpg',
checksum: checksum,
type: AssetType.image,
createdAt: Value(now),
updatedAt: Value(now),
ownerId: ownerId,
visibility: AssetVisibility.timeline,
deletedAt: Value(deletedAt),
),
);
}
Future<void> insertRemoteAssetCloudId({
required String assetId,
required String? cloudId,
DateTime? createdAt,
DateTime? adjustmentTime,
double? latitude,
double? longitude,
}) async {
await db
.into(db.remoteAssetCloudIdEntity)
.insert(
RemoteAssetCloudIdEntityCompanion.insert(
assetId: assetId,
cloudId: Value(cloudId),
createdAt: Value(createdAt),
adjustmentTime: Value(adjustmentTime),
latitude: Value(latitude),
longitude: Value(longitude),
),
);
}
Future<void> insertUser(String id, String email) async {
await db.into(db.userEntity).insert(UserEntityCompanion.insert(id: id, email: email, name: email));
}
group('getRemovalCandidates', () {
final userId = 'user-123';
final otherUserId = 'user-456';
final now = DateTime(2024, 1, 15);
final cutoffDate = DateTime(2024, 1, 10);
final beforeCutoff = DateTime(2024, 1, 5);
final afterCutoff = DateTime(2024, 1, 12);
Future<void> insertUser(String id, String email) async {
await db.into(db.userEntity).insert(UserEntityCompanion.insert(id: id, email: email, name: email));
}
setUp(() async {
await insertUser(userId, 'user@test.com');
await insertUser(otherUserId, 'other@test.com');
});
Future<void> insertLocalAsset({
required String id,
required String checksum,
required DateTime createdAt,
required AssetType type,
required bool isFavorite,
}) async {
await db
.into(db.localAssetEntity)
.insert(
LocalAssetEntityCompanion.insert(
id: id,
name: 'asset_$id.jpg',
checksum: Value(checksum),
type: type,
createdAt: Value(createdAt),
updatedAt: Value(createdAt),
isFavorite: Value(isFavorite),
),
);
}
Future<void> insertRemoteAsset({
required String id,
required String checksum,
required String ownerId,
DateTime? deletedAt,
}) async {
await db
.into(db.remoteAssetEntity)
.insert(
RemoteAssetEntityCompanion.insert(
id: id,
name: 'remote_$id.jpg',
checksum: checksum,
type: AssetType.image,
createdAt: Value(now),
updatedAt: Value(now),
ownerId: ownerId,
visibility: AssetVisibility.timeline,
deletedAt: Value(deletedAt),
),
);
}
Future<void> insertLocalAlbum({required String id, required String name, required bool isIosSharedAlbum}) async {
await db
.into(db.localAlbumEntity)
@@ -211,11 +243,7 @@ void main() {
);
await insertRemoteAsset(id: 'remote-video', checksum: 'checksum-video', ownerId: userId);
final result = await repository.getRemovalCandidates(
userId,
cutoffDate,
keepMediaType: AssetKeepType.photosOnly,
);
final result = await repository.getRemovalCandidates(userId, cutoffDate, keepMediaType: AssetKeepType.photosOnly);
expect(result.assets.length, 1);
expect(result.assets[0].id, 'local-video');
@@ -243,11 +271,7 @@ void main() {
);
await insertRemoteAsset(id: 'remote-video', checksum: 'checksum-video', ownerId: userId);
final result = await repository.getRemovalCandidates(
userId,
cutoffDate,
keepMediaType: AssetKeepType.videosOnly,
);
final result = await repository.getRemovalCandidates(userId, cutoffDate, keepMediaType: AssetKeepType.videosOnly);
expect(result.assets.length, 1);
expect(result.assets[0].id, 'local-photo');
@@ -507,11 +531,7 @@ void main() {
await insertRemoteAsset(id: 'remote-3', checksum: 'checksum-3', ownerId: userId);
await insertLocalAlbumAsset(albumId: 'album-3', assetId: 'local-3');
final result = await repository.getRemovalCandidates(
userId,
cutoffDate,
keepAlbumIds: {'album-1', 'album-2'},
);
final result = await repository.getRemovalCandidates(userId, cutoffDate, keepAlbumIds: {'album-1', 'album-2'});
expect(result.assets.length, 1);
expect(result.assets[0].id, 'local-3');
@@ -644,4 +664,313 @@ void main() {
expect(result.assets[0].id, 'local-video');
});
});
group('reconcileHashesFromCloudId', () {
final userId = 'user-123';
final createdAt = DateTime(2024, 1, 10);
final adjustmentTime = DateTime(2024, 1, 11);
const latitude = 37.7749;
const longitude = -122.4194;
setUp(() async {
await insertUser(userId, 'user@test.com');
});
test('updates local asset checksum when all metadata matches', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, 'hash-abc123');
});
test('does not update when local asset already has checksum', () async {
await insertLocalAsset(
id: 'local-1',
checksum: 'existing-checksum',
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, 'existing-checksum');
});
test('does not update when adjustment_time does not match', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: DateTime(2024, 1, 12),
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('does not update when latitude does not match', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: 40.7128,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('does not update when longitude does not match', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: -74.0060,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('does not update when createdAt does not match', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: DateTime(2024, 1, 5),
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('does not update when iCloudId is null', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: null,
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('does not update when cloudId does not match iCloudId', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-456',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('handles partial null metadata fields matching correctly', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: null,
latitude: latitude,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: null,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, 'hash-abc123');
});
test('does not update when one has null and other has value', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: null,
longitude: longitude,
);
await insertRemoteAsset(id: 'remote-1', checksum: 'hash-abc123', ownerId: userId);
await insertRemoteAssetCloudId(
assetId: 'remote-1',
cloudId: 'cloud-123',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
test('handles no matching assets gracefully', () async {
await insertLocalAsset(
id: 'local-1',
checksum: null,
iCloudId: 'cloud-999',
createdAt: createdAt,
adjustmentTime: adjustmentTime,
latitude: latitude,
longitude: longitude,
);
await repository.reconcileHashesFromCloudId();
final updated = await repository.getById('local-1');
expect(updated?.checksum, isNull);
});
});
}