From e81928678a7206c12db3c52146a0edde065cfb5f Mon Sep 17 00:00:00 2001 From: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Date: Fri, 9 Jan 2026 20:45:25 +0530 Subject: [PATCH] bulk update metadata --- mobile/lib/constants/constants.dart | 2 + .../lib/domain/utils/migrate_cloud_ids.dart | 63 +++++++++++++++++-- .../repositories/local_asset.repository.dart | 36 +++++++++++ .../repositories/sync_stream.repository.dart | 5 +- .../lib/pages/common/splash_screen.page.dart | 8 +-- .../providers/app_life_cycle.provider.dart | 5 +- mobile/lib/services/upload.service.dart | 2 +- 7 files changed, 103 insertions(+), 18 deletions(-) diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index cc408548d2..9d28941b8f 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -4,6 +4,8 @@ const int noDbId = -9223372036854775808; // from Isar const double downloadCompleted = -1; const double downloadFailed = -2; +const String kMobileMetadataKey = "mobile-app"; + // Number of log entries to retain on app start const int kLogTruncateLimit = 2000; diff --git a/mobile/lib/domain/utils/migrate_cloud_ids.dart b/mobile/lib/domain/utils/migrate_cloud_ids.dart index ffafb82e2f..fa761034be 100644 --- a/mobile/lib/domain/utils/migrate_cloud_ids.dart +++ b/mobile/lib/domain/utils/migrate_cloud_ids.dart @@ -1,7 +1,9 @@ import 'package:drift/drift.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/asset/asset_metadata.model.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/extensions/platform_extensions.dart'; import 'package:immich_mobile/infrastructure/entities/local_asset.entity.dart'; import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart'; @@ -9,6 +11,7 @@ import 'package:immich_mobile/platform/native_sync_api.g.dart'; import 'package:immich_mobile/providers/api.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; import 'package:immich_mobile/providers/infrastructure/sync.provider.dart'; +import 'package:immich_mobile/providers/server_info.provider.dart'; import 'package:immich_mobile/providers/user.provider.dart'; import 'package:immich_mobile/utils/debug_print.dart'; import 'package:logging/logging.dart'; @@ -16,10 +19,24 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart' hide AssetVisibility; Future syncCloudIds(ProviderContainer ref) async { + if (!CurrentPlatform.isIOS) { + return; + } + final db = ref.read(driftProvider); // Populate cloud IDs for local assets that don't have one yet await _populateCloudIds(db); + final serverInfo = await ref.read(serverInfoProvider.notifier).getServerInfo(); + final canUpdateMetadata = serverInfo.serverVersion.isAtLeast(major: 2, minor: 4); + if (!canUpdateMetadata) { + Logger( + 'migrateCloudIds', + ).fine('Server version does not support asset metadata updates. Skipping cloudId migration.'); + return; + } + final canBulkUpdateMetadata = serverInfo.serverVersion.isAtLeast(major: 2, minor: 5); + // Wait for remote sync to complete, so we have up-to-date asset metadata entries await ref.read(syncStreamServiceProvider).sync(); @@ -33,9 +50,18 @@ Future syncCloudIds(ProviderContainer ref) async { final mappingsToUpdate = await _fetchCloudIdMappings(db, currentUser.id); dPrint(() => 'Found ${mappingsToUpdate.length} assets to update cloud IDs for.'); final assetApi = ref.read(apiServiceProvider).assetsApi; - for (final mapping in mappingsToUpdate) { - final mobileMeta = AssetMetadataUpsertItemDto( - key: AssetMetadataKey.mobileApp, + + if (canBulkUpdateMetadata) { + await _bulkUpdateCloudIds(assetApi, mappingsToUpdate); + return; + } + await _sequentialUpdateCloudIds(assetApi, mappingsToUpdate); +} + +Future _sequentialUpdateCloudIds(AssetsApi assetsApi, List<_CloudIdMapping> mappings) async { + for (final mapping in mappings) { + final item = AssetMetadataUpsertItemDto( + key: kMobileMetadataKey, value: RemoteAssetMobileAppMetadata( cloudId: mapping.localAsset.cloudId, createdAt: mapping.localAsset.createdAt.toIso8601String(), @@ -45,13 +71,42 @@ Future syncCloudIds(ProviderContainer ref) async { ), ); try { - await assetApi.updateAssetMetadata(mapping.remoteAssetId, AssetMetadataUpsertDto(items: [mobileMeta])); + await assetsApi.updateAssetMetadata(mapping.remoteAssetId, AssetMetadataUpsertDto(items: [item])); } catch (error, stack) { Logger('migrateCloudIds').warning('Failed to update metadata for asset ${mapping.remoteAssetId}', error, stack); } } } +Future _bulkUpdateCloudIds(AssetsApi assetsApi, List<_CloudIdMapping> mappings) async { + const batchSize = 10000; + for (int i = 0; i < mappings.length; i += batchSize) { + final endIndex = (i + batchSize > mappings.length) ? mappings.length : i + batchSize; + final batch = mappings.sublist(i, endIndex); + final items = []; + for (final mapping in batch) { + items.add( + AssetMetadataBulkUpsertItemDto( + assetId: mapping.remoteAssetId, + key: kMobileMetadataKey, + value: RemoteAssetMobileAppMetadata( + cloudId: mapping.localAsset.cloudId, + createdAt: mapping.localAsset.createdAt.toIso8601String(), + adjustmentTime: mapping.localAsset.adjustmentTime?.toIso8601String(), + latitude: mapping.localAsset.latitude?.toString(), + longitude: mapping.localAsset.longitude?.toString(), + ), + ), + ); + } + try { + await assetsApi.updateBulkAssetMetadata(AssetMetadataBulkUpsertDto(items: items)); + } catch (error, stack) { + Logger('migrateCloudIds').warning('Failed to bulk update metadata', error, stack); + } + } +} + Future _populateCloudIds(Drift drift) async { final query = drift.localAssetEntity.selectOnly() ..addColumns([drift.localAssetEntity.id]) diff --git a/mobile/lib/infrastructure/repositories/local_asset.repository.dart b/mobile/lib/infrastructure/repositories/local_asset.repository.dart index 767ec4d39c..6a9181e604 100644 --- a/mobile/lib/infrastructure/repositories/local_asset.repository.dart +++ b/mobile/lib/infrastructure/repositories/local_asset.repository.dart @@ -174,4 +174,40 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository { final rows = await query.get(); return rows.map((row) => row.readTable(_db.localAssetEntity).toDto()).toList(); } + + Future> getEmptyCloudIdAssets() { + final query = _db.localAssetEntity.select()..where((row) => row.iCloudId.isNull()); + return query.map((row) => row.toDto()).get(); + } + + Future> getHashMappingFromCloudId() async { + final query = + _db.localAssetEntity.selectOnly().join([ + leftOuterJoin( + _db.remoteAssetCloudIdEntity, + _db.localAssetEntity.iCloudId.equalsExp(_db.remoteAssetCloudIdEntity.cloudId), + useColumns: false, + ), + leftOuterJoin( + _db.remoteAssetEntity, + _db.remoteAssetCloudIdEntity.assetId.equalsExp(_db.remoteAssetEntity.id), + useColumns: false, + ), + ]) + ..addColumns([_db.localAssetEntity.id, _db.remoteAssetEntity.checksum]) + ..where( + _db.remoteAssetCloudIdEntity.cloudId.isNotNull() & + _db.localAssetEntity.checksum.isNull() & + ((_db.remoteAssetCloudIdEntity.adjustmentTime.isExp(_db.localAssetEntity.adjustmentTime)) & + (_db.remoteAssetCloudIdEntity.latitude.isExp(_db.localAssetEntity.latitude)) & + (_db.remoteAssetCloudIdEntity.longitude.isExp(_db.localAssetEntity.longitude)) & + (_db.remoteAssetCloudIdEntity.createdAt.isExp(_db.localAssetEntity.createdAt))), + ); + final mapping = await query + .map( + (row) => (assetId: row.read(_db.localAssetEntity.id)!, checksum: row.read(_db.remoteAssetEntity.checksum)!), + ) + .get(); + return {for (final entry in mapping) entry.assetId: entry.checksum}; + } } diff --git a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart index b95e76d447..a8f93e7e70 100644 --- a/mobile/lib/infrastructure/repositories/sync_stream.repository.dart +++ b/mobile/lib/infrastructure/repositories/sync_stream.repository.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'package:collection/collection.dart'; import 'package:drift/drift.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/album/album.model.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/domain/models/memory.model.dart'; @@ -264,7 +265,7 @@ class SyncStreamRepository extends DriftDatabaseRepository { try { await _db.batch((batch) { for (final metadata in data) { - if (metadata.key == AssetMetadataKey.mobileApp) { + if (metadata.key == kMobileMetadataKey) { batch.deleteWhere(_db.remoteAssetCloudIdEntity, (row) => row.assetId.equals(metadata.assetId)); } } @@ -279,7 +280,7 @@ class SyncStreamRepository extends DriftDatabaseRepository { try { await _db.batch((batch) { for (final metadata in data) { - if (metadata.key == AssetMetadataKey.mobileApp) { + if (metadata.key == kMobileMetadataKey) { final map = metadata.value as Map; final companion = RemoteAssetCloudIdEntityCompanion( cloudId: Value(map['iCloudId']?.toString()), diff --git a/mobile/lib/pages/common/splash_screen.page.dart b/mobile/lib/pages/common/splash_screen.page.dart index fc24cacc56..6c024600c9 100644 --- a/mobile/lib/pages/common/splash_screen.page.dart +++ b/mobile/lib/pages/common/splash_screen.page.dart @@ -5,13 +5,11 @@ import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; import 'package:immich_mobile/entities/store.entity.dart'; -import 'package:immich_mobile/extensions/platform_extensions.dart'; import 'package:immich_mobile/providers/auth.provider.dart'; import 'package:immich_mobile/providers/background_sync.provider.dart'; import 'package:immich_mobile/providers/backup/backup.provider.dart'; import 'package:immich_mobile/providers/backup/drift_backup.provider.dart'; import 'package:immich_mobile/providers/gallery_permission.provider.dart'; -import 'package:immich_mobile/providers/server_info.provider.dart'; import 'package:immich_mobile/providers/websocket.provider.dart'; import 'package:immich_mobile/routing/router.dart'; import 'package:logging/logging.dart'; @@ -51,7 +49,6 @@ class SplashScreenPageState extends ConsumerState { final accessToken = Store.tryGet(StoreKey.accessToken); if (accessToken != null && serverUrl != null && endpoint != null) { - final infoProvider = ref.read(serverInfoProvider.notifier); final wsProvider = ref.read(websocketProvider.notifier); final backgroundManager = ref.read(backgroundSyncProvider); final backupProvider = ref.read(driftBackupProvider.notifier); @@ -61,7 +58,6 @@ class SplashScreenPageState extends ConsumerState { (_) async { try { wsProvider.connect(); - final serverInfo = await infoProvider.getServerInfo(); if (Store.isBetaTimelineEnabled) { bool syncSuccess = false; @@ -76,9 +72,7 @@ class SplashScreenPageState extends ConsumerState { _resumeBackup(backupProvider); }), _resumeBackup(backupProvider), - // Sync cloud IDs if server version is compatible - if (CurrentPlatform.isIOS && serverInfo.serverVersion.isAtLeast(major: 2, minor: 2)) - backgroundManager.syncCloudIds(), + backgroundManager.syncCloudIds(), ]); } else { await backgroundManager.hashAssets(); diff --git a/mobile/lib/providers/app_life_cycle.provider.dart b/mobile/lib/providers/app_life_cycle.provider.dart index f4e0e5ce32..20ae8d20a3 100644 --- a/mobile/lib/providers/app_life_cycle.provider.dart +++ b/mobile/lib/providers/app_life_cycle.provider.dart @@ -147,7 +147,6 @@ class AppLifeCycleNotifier extends StateNotifier { final backgroundManager = _ref.read(backgroundSyncProvider); final isAlbumLinkedSyncEnable = _ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.syncAlbums); - final serverInfo = _ref.read(serverInfoProvider); try { bool syncSuccess = false; @@ -161,9 +160,7 @@ class AppLifeCycleNotifier extends StateNotifier { _resumeBackup(); }), _resumeBackup(), - // Sync cloud IDs if server version is compatible - if (CurrentPlatform.isIOS && serverInfo.serverVersion.isAtLeast(major: 2, minor: 2)) - backgroundManager.syncCloudIds(), + backgroundManager.syncCloudIds(), ]); } else { await _safeRun(backgroundManager.hashAssets(), "hashAssets"); diff --git a/mobile/lib/services/upload.service.dart b/mobile/lib/services/upload.service.dart index 73c0c943e6..f4ee73ad41 100644 --- a/mobile/lib/services/upload.service.dart +++ b/mobile/lib/services/upload.service.dart @@ -444,7 +444,7 @@ class UploadService { 'duration': '0', if (fields != null) ...fields, // Include cloudId and eTag in metadata if available and server version supports it - if (CurrentPlatform.isIOS && cloudId != null && _serverInfo.serverVersion.isAtLeast(major: 2, minor: 2)) + if (CurrentPlatform.isIOS && cloudId != null && _serverInfo.serverVersion.isAtLeast(major: 2, minor: 4)) 'metadata': jsonEncode([ RemoteAssetMetadataItem( key: RemoteAssetMetadataKey.mobileApp,