fix: automatic delete should not break sync

.github/workflows/cli.yml

@@ -78,10 +78,10 @@ jobs:
         uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
         if: ${{ !github.event.pull_request.head.repo.fork }}
         with:
           registry: ghcr.io
@@ -96,7 +96,7 @@ jobs:
 
       - name: Generate docker image tags
         id: metadata
-        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
         with:
           flavor: |
             latest=false
@@ -107,7 +107,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }}
 
       - name: Build and push image
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
         with:
           file: packages/cli/Dockerfile
           platforms: linux/amd64,linux/arm64

.github/workflows/codeql-analysis.yml

@@ -57,7 +57,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
+        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -70,7 +70,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
+        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -83,6 +83,6 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
+        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
         with:
           category: '/language:${{matrix.language}}'

.github/workflows/docker.yml

@@ -60,7 +60,7 @@ jobs:
         suffix: ['', '-cuda', '-rocm', '-openvino', '-armnn', '-rknn']
     steps:
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -90,7 +90,7 @@ jobs:
         suffix: ['']
     steps:
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

mobile/lib/domain/services/local_sync.service.dart

@@ -413,7 +413,16 @@ class LocalSyncService {
     if (localAssetsToTrash.isNotEmpty) {
       final localIds = localAssetsToTrash.values.expand((assets) => assets).map((asset) => asset.id).toList();
       _log.info("Moving to trash ${localIds.join(", ")} assets");
-      final movedIds = await _assetMediaRepository.deleteAll(localIds);
+      final List<String> movedIds;
+      try {
+        movedIds = await _assetMediaRepository.deleteAll(localIds);
+      } catch (error, stackTrace) {
+        _log.severe("Failed to move local assets to trash", error, stackTrace);
+        // Automatic trash sync shouldn't block sync of other assets
+        // If the delete fails because an asset is in trash, the successive sync process
+        // will also remove it from the local DB
+        return;
+      }
       if (movedIds.isNotEmpty) {
         final movedAssetsByAlbum = localAssetsToTrash.map(
           (albumId, assets) => MapEntry(albumId, assets.where((asset) => movedIds.contains(asset.id)).toList()),

mobile/lib/domain/services/sync_stream.service.dart

@@ -516,7 +516,14 @@ class SyncStreamService {
   Future<void> _trashLocalAssets(Map<String, List<LocalAsset>> localAssetsToTrash) async {
     final localIds = localAssetsToTrash.values.expand((assets) => assets).map((asset) => asset.id).toList();
     _logger.info("Moving to trash ${localIds.join(", ")} assets");
-    final movedIds = await _assetMediaRepository.deleteAll(localIds);
+    final List<String> movedIds;
+    try {
+      movedIds = await _assetMediaRepository.deleteAll(localIds);
+    } catch (error, stackTrace) {
+      _logger.severe("Failed to move local assets to trash", error, stackTrace);
+      // The local sync service retries this on the next sync, so it is safe to log and exit early
+      return;
+    }
     if (movedIds.isNotEmpty) {
       final movedAssetsByAlbum = localAssetsToTrash.map(
         (albumId, assets) => MapEntry(albumId, assets.where((asset) => movedIds.contains(asset.id)).toList()),

mobile/lib/repositories/asset_media.repository.dart

@@ -45,6 +45,7 @@ class AssetMediaRepository {
     return false;
   }
 
+  // TODO: Replace the calls with a native pigeon method that filters ids no longer in the MediaStore before building the request
   Future<List<String>> deleteAll(List<String> ids) async {
     if (CurrentPlatform.isAndroid) {
       if (await _androidSupportsTrash()) {

mobile/openapi/lib/model/system_config_integrity_checksum_job.dart

@@ -28,8 +28,8 @@ class SystemConfigIntegrityChecksumJob {
   /// Percentage limit of the integrity checksum job
   ///
   /// Minimum value: 0
-  /// Maximum value: 1
-  double percentageLimit;
+  /// Maximum value: 9007199254740991
+  int percentageLimit;
 
   /// How long the integrity checksum job may run for
   ///
@@ -75,7 +75,7 @@ class SystemConfigIntegrityChecksumJob {
       return SystemConfigIntegrityChecksumJob(
         cronExpression: mapValueOfType<String>(json, r'cronExpression')!,
         enabled: mapValueOfType<bool>(json, r'enabled')!,
-        percentageLimit: mapValueOfType<double>(json, r'percentageLimit')!,
+        percentageLimit: mapValueOfType<int>(json, r'percentageLimit')!,
         timeLimit: mapValueOfType<int>(json, r'timeLimit')!,
       );
     }

mobile/pubspec.lock

@@ -309,8 +309,8 @@ packages:
     dependency: "direct main"
     description:
       path: "pkgs/cupertino_http"
-      ref: "58b03c756b81d16b3975a8ae4b91d25360123bb5"
-      resolved-ref: "58b03c756b81d16b3975a8ae4b91d25360123bb5"
+      ref: a0a933358517c6d01cff37fc2a2752ee2d744a3c
+      resolved-ref: a0a933358517c6d01cff37fc2a2752ee2d744a3c
       url: "https://github.com/mertalev/http"
     source: git
     version: "3.0.0-wip"
@@ -1158,13 +1158,12 @@ packages:
     source: hosted
     version: "0.5.0"
   objective_c:
-    dependency: "direct overridden"
+    dependency: transitive
     description:
-      path: "pkgs/objective_c"
-      ref: "2915556701f4734a784222f290a82adb1b101682"
-      resolved-ref: "2915556701f4734a784222f290a82adb1b101682"
-      url: "https://github.com/mertalev/native"
-    source: git
+      name: objective_c
+      sha256: "6cb691c686fa2838c6deb34980d426145c2a5d537491cb83d463c33cdbc726ed"
+      url: "https://pub.dev"
+    source: hosted
     version: "9.4.1"
   octo_image:
     dependency: "direct main"

mobile/pubspec.yaml

@@ -84,7 +84,7 @@ dependencies:
   cupertino_http:
     git:
       url: https://github.com/mertalev/http
-      ref: '58b03c756b81d16b3975a8ae4b91d25360123bb5'
+      ref: 'a0a933358517c6d01cff37fc2a2752ee2d744a3c' # https://github.com/dart-lang/http/pull/1876
       path: pkgs/cupertino_http/
   ok_http:
     git:
@@ -114,11 +114,6 @@ dev_dependencies:
 # Pin bonsoir to 5.x until cast releases a version compatible with bonsoir 6.x.
 dependency_overrides:
   bonsoir: ^5.1.11
-  objective_c:
-    git:
-      url: https://github.com/mertalev/native
-      ref: '2915556701f4734a784222f290a82adb1b101682' # https://github.com/dart-lang/native/pull/3458
-      path: pkgs/objective_c/
 
 flutter:
   uses-material-design: true

mobile/test/domain/services/local_sync_service_test.dart

@@ -1,6 +1,7 @@
 import 'package:drift/drift.dart' as drift;
 import 'package:drift/native.dart';
 import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/domain/models/store.model.dart';
@@ -197,6 +198,24 @@ void main() {
       expect(trashArgs['album-a'], [movedAsset]);
     });
 
+    test('continues the device diff when moving local assets to trash fails', () async {
+      await Store.put(StoreKey.manageLocalMediaAndroid, true);
+      when(() => mockPermissionRepository.hasManageMediaPermission()).thenAnswer((_) async => true);
+      when(() => mockTrashedLocalAssetRepository.getToTrash()).thenAnswer(
+        (_) async => {
+          'album-a': [LocalAssetStub.image1],
+        },
+      );
+      when(
+        () => mockAssetMediaRepository.deleteAll(any()),
+      ).thenThrow(PlatformException(code: 'PLATFORM_ERROR', message: 'deleteWithIds failed'));
+
+      await sut.sync();
+
+      verifyNever(() => mockTrashedLocalAssetRepository.trashLocalAsset(any()));
+      verify(() => mockNativeSyncApi.getMediaChanges()).called(1);
+    });
+
     test('does not attempt restore when repository has no assets to restore', () async {
       when(() => mockTrashedLocalAssetRepository.getToRestore()).thenAnswer((_) async => []);
 

mobile/test/domain/services/sync_stream_service_test.dart

@@ -3,6 +3,7 @@ import 'dart:async';
 import 'package:drift/drift.dart' as drift;
 import 'package:drift/native.dart';
 import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/domain/models/store.model.dart';
@@ -36,7 +37,6 @@ class _AbortCallbackWrapper {
 
 class _MockAbortCallbackWrapper extends Mock implements _AbortCallbackWrapper {}
 
-
 void main() {
   late SyncStreamService sut;
   late SyncStreamRepository mockSyncStreamRepo;
@@ -494,6 +494,32 @@ void main() {
       expect(trashArgs['album-a'], [movedAsset]);
     });
 
+    test("acks the batch and skips trash bookkeeping when moving assets to device trash fails", () async {
+      final localAsset = LocalAssetStub.image1.copyWith(id: 'stale-local', checksum: 'checksum-stale');
+      when(() => mockLocalAssetRepo.getAssetsFromBackupAlbums(any())).thenAnswer(
+        (_) async => {
+          'album-a': [localAsset],
+        },
+      );
+      when(
+        () => mockAssetMediaRepo.deleteAll(any()),
+      ).thenThrow(PlatformException(code: 'PLATFORM_ERROR', message: 'deleteWithIds failed'));
+
+      final events = [
+        SyncStreamStub.assetTrashed(
+          id: 'remote-1',
+          checksum: localAsset.checksum!,
+          ack: 'asset-remote-1',
+          trashedAt: DateTime(2025, 5, 1),
+        ),
+      ];
+
+      await simulateEvents(events);
+
+      verifyNever(() => mockTrashedLocalAssetRepo.trashLocalAsset(any()));
+      verify(() => mockSyncApiRepo.ack(['asset-remote-1'])).called(1);
+    });
+
     test("skips device trashing when no local assets match the remote trash payload", () async {
       final events = [
         SyncStreamStub.assetTrashed(

open-api/immich-openapi-specs.json

@@ -25917,10 +25917,9 @@
           },
           "percentageLimit": {
             "description": "Percentage limit of the integrity checksum job",
-            "format": "double",
-            "maximum": 1,
+            "maximum": 9007199254740991,
             "minimum": 0,
-            "type": "number"
+            "type": "integer"
           },
           "timeLimit": {
             "description": "How long the integrity checksum job may run for",

server/src/dtos/system-config.dto.ts

@@ -74,12 +74,7 @@ const SystemConfigIntegrityJobSchema = z
 
 const SystemConfigIntegrityChecksumJobSchema = SystemConfigIntegrityJobSchema.extend({
   timeLimit: z.int().nonnegative().describe('How long the integrity checksum job may run for'),
-  percentageLimit: z
-    .float32()
-    .nonnegative()
-    .max(1)
-    .describe('Percentage limit of the integrity checksum job')
-    .meta({ format: 'double' }),
+  percentageLimit: z.int().nonnegative().describe('Percentage limit of the integrity checksum job'),
 })
   .describe('Integrity checksum job config')
   .meta({ id: 'SystemConfigIntegrityChecksumJob' });

server/src/repositories/media.repository.ts

@@ -3,9 +3,10 @@ import { ExifDateTime, exiftool, WriteTags } from 'exiftool-vendored';
 import ffmpeg, { FfprobeData, FfprobeStream } from 'fluent-ffmpeg';
 import _ from 'lodash';
 import { Duration } from 'luxon';
-import { spawn } from 'node:child_process';
+import { execFile as execFileCb } from 'node:child_process';
 import fs from 'node:fs/promises';
 import { Writable } from 'node:stream';
+import { promisify } from 'node:util';
 import sharp from 'sharp';
 import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants';
 import { Exif } from 'src/database';
@@ -43,6 +44,8 @@ const probe = (input: string, options: string[]): Promise<FfprobeData> =>
     ffmpeg.ffprobe(input, options, (error, data) => (error ? reject(error) : resolve(data))),
   );
 
+const execFile = promisify(execFileCb);
+
 sharp.concurrency(0);
 sharp.cache({ files: 0 });
 
@@ -288,37 +291,33 @@ export class MediaRepository {
    * Needed for accurate segments, especially when remuxing, seeking and/or VFR is involved.
    * Scanning packets for keyframes in JS is much faster than -skip_frame nokey since it avoids decoding the video.
    */
-  probePackets(input: string, streamIndex: number): Promise<VideoPacketInfo | null> {
-    const ffprobe = spawn(
-      'ffprobe',
-      [
-        '-v',
-        'error',
-        '-select_streams',
-        String(streamIndex),
-        '-show_entries',
-        'packet=pts,duration,flags',
-        '-of',
-        'csv=p=0',
-        input,
-      ],
-      { stdio: ['ignore', 'pipe', 'pipe'] },
-    );
+  async probePackets(input: string, streamIndex: number): Promise<VideoPacketInfo | null> {
+    const { stdout } = await execFile('ffprobe', [
+      '-v',
+      'error',
+      '-select_streams',
+      String(streamIndex),
+      '-show_entries',
+      'packet=pts,duration,flags',
+      '-of',
+      'csv=p=0',
+      input,
+    ]);
 
     let totalDuration = 0;
     const keyframePts: number[] = [];
     const keyframeAccDuration: number[] = [];
     const keyframeOwnDuration: number[] = [];
     const postDiscard: { pts: number; duration: number }[] = [];
-    const parseLine = (line: string) => {
+    for (const line of stdout.split('\n')) {
       if (!line) {
-        return;
+        continue;
       }
       const [ptsStr, durationStr, flags] = line.split(',');
       const pts = Number.parseInt(ptsStr);
       const duration = Number.parseInt(durationStr);
-      if (Number.isNaN(pts) || Number.isNaN(duration) || !flags) {
-        return;
+      if (Number.isNaN(pts) || Number.isNaN(duration)) {
+        continue;
       }
       // Discarded packets don't contribute to packet count, but still contribute to video duration
       totalDuration += duration;
@@ -333,43 +332,20 @@ export class MediaRepository {
         // Non-keyframes are accounted for in totalDuration.
         keyframeOwnDuration.push(duration);
       }
+    }
+
+    if (postDiscard.length === 0) {
+      return null;
+    }
+
+    return {
+      totalDuration,
+      packetCount: postDiscard.length,
+      outputFrames: this.cfrOutputFrames(postDiscard, postDiscard.length / totalDuration),
+      keyframePts,
+      keyframeAccDuration,
+      keyframeOwnDuration,
     };
-
-    let stderr = '';
-    let remainder = '';
-    ffprobe.stderr.setEncoding('utf8');
-    ffprobe.stderr.on('data', (chunk: string) => (stderr += chunk));
-    ffprobe.stdout.setEncoding('utf8');
-    ffprobe.stdout.on('data', (chunk: string) => {
-      const lines = chunk.split('\n');
-      lines[0] = remainder + lines[0];
-      remainder = lines.pop() as string;
-      for (const line of lines) {
-        parseLine(line);
-      }
-    });
-
-    return new Promise<VideoPacketInfo | null>((resolve, reject) => {
-      ffprobe.on('error', reject);
-      ffprobe.on('close', (code) => {
-        if (code !== 0) {
-          return reject(new Error(`ffprobe exited with code ${code}: ${stderr.trim()}`));
-        }
-        parseLine(remainder);
-        if (postDiscard.length === 0) {
-          return resolve(null);
-        }
-
-        resolve({
-          totalDuration,
-          packetCount: postDiscard.length,
-          outputFrames: this.cfrOutputFrames(postDiscard, postDiscard.length / totalDuration),
-          keyframePts,
-          keyframeAccDuration,
-          keyframeOwnDuration,
-        });
-      });
-    });
   }
 
   transcode(input: string, output: string | Writable, options: TranscodeCommand): Promise<void> {

server/src/services/database-backup.service.ts

@@ -407,7 +407,7 @@ export class DatabaseBackupService {
         progressCb?.('restore', progress);
       });
 
-      await pipeline(sqlStream, createSqlOwnerTransformStream(databaseUsername), progressSource, psql, progressSink);
+      await pipeline(sqlStream, progressSource, psql, progressSink);
 
       try {
         progressCb?.('migrations', 0.9);
@@ -572,69 +572,3 @@ function createSqlProgressStreams(cb: (progress: number) => void) {
 
   return [source, sink];
 }
-
-function createSqlOwnerTransformStream(databaseUsername: string) {
-  const OWNER_MARKER_START = new TextEncoder().encode('OWNER TO ');
-  const DATA_MARKER_START = new TextEncoder().encode('FROM stdin');
-  const LINE_END = new TextEncoder().encode(';');
-
-  const owner = new TextEncoder().encode(databaseUsername);
-
-  let ownerSequenceIndex = 0;
-
-  let replacingOwnerIndex = 0;
-  let replacingOwner = false;
-
-  let readingDataIndex = 0;
-
-  let dataPart = false;
-
-  return new PassThrough({
-    transform(chunk, _encoding, callback) {
-      let result = chunk;
-      if (!dataPart) {
-        for (let index = 0; index < result.length; index++) {
-          if (replacingOwner) {
-            if (result[index] === LINE_END[0]) {
-              result = Buffer.concat([result.slice(0, index), owner.slice(replacingOwnerIndex), result.slice(index)]);
-              replacingOwnerIndex = owner.length;
-            } else {
-              result[index] = owner[replacingOwnerIndex];
-              replacingOwnerIndex++;
-            }
-          }
-
-          if (replacingOwnerIndex === owner.length) {
-            replacingOwner = false;
-          }
-
-          if (result[index] === OWNER_MARKER_START[ownerSequenceIndex]) {
-            ownerSequenceIndex++;
-          } else {
-            ownerSequenceIndex = 0;
-          }
-
-          if (ownerSequenceIndex === OWNER_MARKER_START.length) {
-            ownerSequenceIndex = 0;
-            replacingOwner = true;
-            replacingOwnerIndex = 0;
-          }
-
-          if (result[index] === DATA_MARKER_START[readingDataIndex]) {
-            readingDataIndex++;
-          } else {
-            readingDataIndex = 0;
-          }
-
-          if (readingDataIndex === DATA_MARKER_START.length) {
-            dataPart = true;
-            break;
-          }
-        }
-      }
-
-      this.push(result);
-      callback();
-    },
-  });
-}