From ca660c5abe0dbdf9f81778ad37504809cff0addf Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 17 Jan 2026 23:46:46 -0500 Subject: [PATCH] flutter-side decode --- .../android/app/src/main/cpp/native_buffer.c | 25 ++-- .../alextran/immich/images/LocalImagesImpl.kt | 9 +- .../immich/images/RemoteImagesImpl.kt | 107 ++++++++---------- mobile/ios/Podfile.lock | 6 - .../infrastructure/loaders/image_request.dart | 67 +++++++---- .../loaders/local_image_request.dart | 2 +- .../loaders/remote_image_request.dart | 11 +- .../loaders/thumbhash_image_request.dart | 2 +- 8 files changed, 106 insertions(+), 123 deletions(-) diff --git a/mobile/android/app/src/main/cpp/native_buffer.c b/mobile/android/app/src/main/cpp/native_buffer.c index dfeda4c427..ab21f8b351 100644 --- a/mobile/android/app/src/main/cpp/native_buffer.c +++ b/mobile/android/app/src/main/cpp/native_buffer.c @@ -1,6 +1,5 @@ #include #include -#include JNIEXPORT jlong JNICALL Java_app_alextran_immich_images_LocalImagesImpl_allocateNative( @@ -15,25 +14,15 @@ Java_app_alextran_immich_images_LocalImagesImpl_freeNative( free((void *) address); } +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_images_LocalImagesImpl_reallocNative( + JNIEnv *env, jclass clazz, jlong address, jint size) { + void *ptr = realloc((void *) address, size); + return (jlong) ptr; +} + JNIEXPORT jobject JNICALL Java_app_alextran_immich_images_LocalImagesImpl_wrapAsBuffer( JNIEnv *env, jclass clazz, jlong address, jint capacity) { return (*env)->NewDirectByteBuffer(env, (void *) address, capacity); } - -JNIEXPORT jlong JNICALL -Java_app_alextran_immich_images_RemoteImagesImpl_lockBitmapPixels( - JNIEnv *env, jclass clazz, jobject bitmap) { - void *pixels = NULL; - int result = AndroidBitmap_lockPixels(env, bitmap, &pixels); - if (result != ANDROID_BITMAP_RESULT_SUCCESS) { - return 0; - } - return (jlong) pixels; -} - -JNIEXPORT void JNICALL -Java_app_alextran_immich_images_RemoteImagesImpl_unlockBitmapPixels( - JNIEnv *env, jclass clazz, jobject bitmap) { - AndroidBitmap_unlockPixels(env, bitmap); -} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt index 6c13bc40bb..a37a819928 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt @@ -52,7 +52,8 @@ fun Bitmap.toNativeBuffer(): Map { return mapOf( "pointer" to pointer, "width" to width.toLong(), - "height" to height.toLong() + "height" to height.toLong(), + "rowBytes" to (width * 4).toLong() ) } catch (e: Exception) { LocalImagesImpl.freeNative(pointer) @@ -83,6 +84,9 @@ class LocalImagesImpl(context: Context) : LocalImageApi { @JvmStatic external fun freeNative(pointer: Long) + @JvmStatic + external fun reallocNative(pointer: Long, size: Int): Long + @JvmStatic external fun wrapAsBuffer(address: Long, capacity: Int): ByteBuffer } @@ -95,7 +99,8 @@ class LocalImagesImpl(context: Context) : LocalImageApi { val res = mapOf( "pointer" to image.pointer, "width" to image.width.toLong(), - "height" to image.height.toLong() + "height" to image.height.toLong(), + "rowBytes" to (image.width * 4).toLong() ) callback(Result.success(res)) } catch (e: Exception) { diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt index 42eed0f4ba..879eeda7a5 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt @@ -1,11 +1,6 @@ package app.alextran.immich.images import android.content.Context -import android.graphics.Bitmap -import android.graphics.BitmapFactory -import android.graphics.ColorSpace -import android.graphics.ImageDecoder -import android.os.Build import android.os.CancellationSignal import app.alextran.immich.BuildConfig import app.alextran.immich.core.SSLConfig @@ -21,15 +16,14 @@ import okhttp3.Dispatcher import org.chromium.net.CronetEngine import java.io.File import java.io.IOException -import java.nio.ByteBuffer import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import okhttp3.Interceptor data class RemoteRequest( val callback: (Result>) -> Unit, val cancellationSignal: CancellationSignal, + var pointer: Long = 0L, ) class UserAgentInterceptor : Interceptor { @@ -48,7 +42,6 @@ class UserAgentInterceptor : Interceptor { class RemoteImagesImpl(context: Context) : RemoteImageApi { private val requestMap = ConcurrentHashMap() - private val lockedBitmaps = ConcurrentHashMap() init { appContext = context.applicationContext @@ -62,9 +55,9 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { private const val KEEP_ALIVE_DURATION_MINUTES = 5L private const val CACHE_SIZE_BYTES = 1024L * 1024 * 1024 + private const val INITIAL_BUFFER_SIZE = 64 * 1024 + val CANCELLED = Result.success>(emptyMap()) - private val decodePool = - Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1) private var appContext: Context? = null private var cacheDir: File? = null @@ -76,12 +69,6 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { SSLConfig.addListener(::invalidateClient) } - @JvmStatic - external fun lockBitmapPixels(bitmap: Bitmap): Long - - @JvmStatic - external fun unlockBitmapPixels(bitmap: Bitmap) - private fun invalidateClient() { (client as? OkHttpClient)?.let { it.dispatcher.cancelAll() @@ -111,7 +98,7 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { .enableQuic(true) .enableBrotli(true) .setStoragePath(storageDir.absolutePath) - .enableHttpCache(CronetEngine.Builder.HTTP_CACHE_DISK, CACHE_SIZE_BYTES) +// .enableHttpCache(CronetEngine.Builder.HTTP_CACHE_DISK, CACHE_SIZE_BYTES) .setUserAgent(UserAgentInterceptor.USER_AGENT) .build() .also { cronetEngine = it } @@ -168,60 +155,56 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi { } override fun onResponse(call: Call, response: Response) { - decodePool.execute { - try { - signal.throwIfCanceled() - val bytes = response.takeIf { it.isSuccessful }?.body?.bytes() - ?: return@execute callback(Result.failure(IOException(response.toString()))) - signal.throwIfCanceled() - val bitmap = decodeImage(bytes) - signal.throwIfCanceled() + var pointer = 0L + var capacity: Int + try { + signal.throwIfCanceled() + val body = response.takeIf { it.isSuccessful }?.body + ?: return callback(Result.failure(IOException(response.toString()))) - val pointer = lockBitmapPixels(bitmap) - if (pointer == 0L) { - bitmap.recycle() - return@execute callback(Result.failure(RuntimeException("Failed to lock bitmap pixels"))) + val contentLength = body.contentLength() + capacity = if (contentLength > 0) contentLength.toInt() else INITIAL_BUFFER_SIZE + pointer = LocalImagesImpl.allocateNative(capacity) + request.pointer = pointer + + var position = 0 + body.source().use { source -> + while (!source.exhausted()) { + signal.throwIfCanceled() + if (position >= capacity) { + capacity = maxOf(capacity * 2, position + 8192) + pointer = LocalImagesImpl.reallocNative(pointer, capacity) + request.pointer = pointer + } + val buffer = LocalImagesImpl.wrapAsBuffer(pointer + position, capacity - position) + val read = source.read(buffer) + if (read == -1) break + position += read } - - lockedBitmaps[requestId] = bitmap - callback(Result.success(mapOf( - "pointer" to pointer, - "width" to bitmap.width.toLong(), - "height" to bitmap.height.toLong(), - "rowBytes" to bitmap.rowBytes.toLong() - ))) - } catch (e: Exception) { - val result = if (signal.isCanceled) CANCELLED else Result.failure(e) - callback(result) - } finally { - requestMap.remove(requestId) - response.close() } + + signal.throwIfCanceled() + request.pointer = 0L // Transfer ownership to Dart before callback + callback(Result.success(mapOf( + "pointer" to pointer, + "length" to position.toLong() + ))) + } catch (e: Exception) { + if (pointer != 0L) LocalImagesImpl.freeNative(pointer) + val result = if (signal.isCanceled) CANCELLED else Result.failure(e) + callback(result) + } finally { + requestMap.remove(requestId) + response.close() } } }) } override fun cancelRequest(requestId: Long) { - requestMap.remove(requestId)?.cancellationSignal?.cancel() - releaseImage(requestId) + // Just cancel the signal - memory cleanup happens in onResponse/onFailure + requestMap[requestId]?.cancellationSignal?.cancel() } - override fun releaseImage(requestId: Long) { - val bitmap = lockedBitmaps.remove(requestId) ?: return - unlockBitmapPixels(bitmap) - bitmap.recycle() - } - - private fun decodeImage(bytes: ByteArray): Bitmap { - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - ImageDecoder.createSource(ByteBuffer.wrap(bytes)).decodeBitmap() - } else { - val options = BitmapFactory.Options().apply { - inPreferredConfig = Bitmap.Config.ARGB_8888 - inPreferredColorSpace = ColorSpace.get(ColorSpace.Named.SRGB) - } - BitmapFactory.decodeByteArray(bytes, 0, bytes.size, options) - } - } + override fun releaseImage(requestId: Long) {} } diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 77caaeceef..172d035cd3 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -80,8 +80,6 @@ PODS: - Flutter - network_info_plus (0.0.1): - Flutter - - objective_c (0.0.1): - - Flutter - package_info_plus (0.4.5): - Flutter - path_provider_foundation (0.0.1): @@ -160,7 +158,6 @@ DEPENDENCIES: - maplibre_gl (from `.symlinks/plugins/maplibre_gl/ios`) - native_video_player (from `.symlinks/plugins/native_video_player/ios`) - network_info_plus (from `.symlinks/plugins/network_info_plus/ios`) - - objective_c (from `.symlinks/plugins/objective_c/ios`) - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`) - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`) - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`) @@ -229,8 +226,6 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/native_video_player/ios" network_info_plus: :path: ".symlinks/plugins/network_info_plus/ios" - objective_c: - :path: ".symlinks/plugins/objective_c/ios" package_info_plus: :path: ".symlinks/plugins/package_info_plus/ios" path_provider_foundation: @@ -282,7 +277,6 @@ SPEC CHECKSUMS: maplibre_gl: 3c924e44725147b03dda33430ad216005b40555f native_video_player: b65c58951ede2f93d103a25366bdebca95081265 network_info_plus: cf61925ab5205dce05a4f0895989afdb6aade5fc - objective_c: 89e720c30d716b036faf9c9684022048eee1eee2 package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499 path_provider_foundation: bb55f6dbba17d0dccd6737fe6f7f34fbd0376880 permission_handler_apple: 4ed2196e43d0651e8ff7ca3483a069d469701f2d diff --git a/mobile/lib/infrastructure/loaders/image_request.dart b/mobile/lib/infrastructure/loaders/image_request.dart index 14ab78ba09..901e8be48a 100644 --- a/mobile/lib/infrastructure/loaders/image_request.dart +++ b/mobile/lib/infrastructure/loaders/image_request.dart @@ -1,6 +1,5 @@ import 'dart:async'; import 'dart:ffi'; -import 'dart:io'; import 'dart:ui' as ui; import 'package:flutter/material.dart'; @@ -35,35 +34,55 @@ abstract class ImageRequest { void _onCancelled(); - Future _fromPlatformImage(Map info, {required bool shouldFree}) async { - final address = info['pointer']; - if (address == null) { - return null; - } - + Future _fromEncodedPlatformImage(int address, int length) async { final pointer = Pointer.fromAddress(address); if (_isCancelled) { - if (shouldFree) { - malloc.free(pointer); - } + malloc.free(pointer); return null; } - final int actualWidth; - final int actualHeight; - final int rowBytes; - final int actualSize; final ui.ImmutableBuffer buffer; try { - actualWidth = info['width']!; - actualHeight = info['height']!; - rowBytes = info['rowBytes'] ?? actualWidth * 4; - actualSize = rowBytes * actualHeight; - buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); + buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(length)); } finally { - if (shouldFree) { - malloc.free(pointer); - } + malloc.free(pointer); + } + + if (_isCancelled) { + buffer.dispose(); + return null; + } + + final descriptor = await ui.ImageDescriptor.encoded(buffer); + if (_isCancelled) { + buffer.dispose(); + descriptor.dispose(); + return null; + } + + final codec = await descriptor.instantiateCodec(); + if (_isCancelled) { + buffer.dispose(); + descriptor.dispose(); + codec.dispose(); + return null; + } + return await codec.getNextFrame(); + } + + Future _fromDecodedPlatformImage(int address, int width, int height, int rowBytes) async { + final pointer = Pointer.fromAddress(address); + if (_isCancelled) { + malloc.free(pointer); + return null; + } + + final size = rowBytes * height; + final ui.ImmutableBuffer buffer; + try { + buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(size)); + } finally { + malloc.free(pointer); } if (_isCancelled) { @@ -73,8 +92,8 @@ abstract class ImageRequest { final descriptor = ui.ImageDescriptor.raw( buffer, - width: actualWidth, - height: actualHeight, + width: width, + height: height, rowBytes: rowBytes, pixelFormat: ui.PixelFormat.rgba8888, ); diff --git a/mobile/lib/infrastructure/loaders/local_image_request.dart b/mobile/lib/infrastructure/loaders/local_image_request.dart index 863a5cd802..ce8c3debfd 100644 --- a/mobile/lib/infrastructure/loaders/local_image_request.dart +++ b/mobile/lib/infrastructure/loaders/local_image_request.dart @@ -24,7 +24,7 @@ class LocalImageRequest extends ImageRequest { isVideo: assetType == AssetType.video, ); - final frame = await _fromPlatformImage(info, shouldFree: true); + final frame = await _fromDecodedPlatformImage(info["pointer"]!, info["width"]!, info["height"]!, info["rowBytes"]!); return frame == null ? null : ImageInfo(image: frame.image, scale: scale); } diff --git a/mobile/lib/infrastructure/loaders/remote_image_request.dart b/mobile/lib/infrastructure/loaders/remote_image_request.dart index b306ca272d..7840551e50 100644 --- a/mobile/lib/infrastructure/loaders/remote_image_request.dart +++ b/mobile/lib/infrastructure/loaders/remote_image_request.dart @@ -13,15 +13,8 @@ class RemoteImageRequest extends ImageRequest { } final Map info = await remoteImageApi.requestImage(uri, headers: headers, requestId: requestId); - - try { - final frame = await _fromPlatformImage(info, shouldFree: Platform.isIOS); - return frame == null ? null : ImageInfo(image: frame.image, scale: scale); - } finally { - if (Platform.isAndroid) { - unawaited(remoteImageApi.releaseImage(requestId)); - } - } + final frame = await _fromEncodedPlatformImage(info["pointer"]!, info["length"]!); + return frame == null ? null : ImageInfo(image: frame.image, scale: scale); } @override diff --git a/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart b/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart index 4cf7d545b5..2ced28b810 100644 --- a/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart +++ b/mobile/lib/infrastructure/loaders/thumbhash_image_request.dart @@ -12,7 +12,7 @@ class ThumbhashImageRequest extends ImageRequest { } final Map info = await localImageApi.getThumbhash(thumbhash); - final frame = await _fromPlatformImage(info, shouldFree: true); + final frame = await _fromDecodedPlatformImage(info["pointer"]!, info["width"]!, info["height"]!, info["rowBytes"]!); return frame == null ? null : ImageInfo(image: frame.image, scale: scale); }