update revision

mobile/ios/Runner/Core/URLSessionManager.swift

@@ -67,6 +67,9 @@ class URLSessionManager: NSObject {
     delegate = URLSessionManagerDelegate()
     session = Self.buildSession(delegate: delegate)
     super.init()
+    if #available(iOS 15, *) {
+      VideoProxyServer.shared.session = session
+    }
     Self.serverUrls = UserDefaults.group.stringArray(forKey: SERVER_URLS_KEY) ?? []
     NotificationCenter.default.addObserver(
       Self.self,
@@ -78,6 +81,9 @@ class URLSessionManager: NSObject {
 
   func recreateSession() {
     session = Self.buildSession(delegate: delegate)
+    if #available(iOS 15, *) {
+      VideoProxyServer.shared.session = session
+    }
   }
 
   static func setServerUrls(_ urls: [String]) {
@@ -249,9 +255,6 @@ class URLSessionManagerDelegate: NSObject, URLSessionTaskDelegate, URLSessionWeb
       let credential = URLCredential(identity: identity as! SecIdentity,
                                      certificates: nil,
                                      persistence: .forSession)
-      if #available(iOS 15, *) {
-        VideoProxyServer.shared.session = session
-      }
       return completion(.useCredential, credential)
     }
     completion(.performDefaultHandling, nil)
@@ -268,9 +271,6 @@ class URLSessionManagerDelegate: NSObject, URLSessionTaskDelegate, URLSessionWeb
     else {
       return completion(.performDefaultHandling, nil)
     }
-    if #available(iOS 15, *) {
-      VideoProxyServer.shared.session = session
-    }
     let credential = URLCredential(user: user, password: password, persistence: .forSession)
     completion(.useCredential, credential)
   }

mobile/lib/models/server_info/server_features.model.dart

@@ -7,6 +7,7 @@ class ServerFeatures {
   final bool passwordLogin;
   final bool ocr;
   final bool smartSearch;
+  final bool realtimeTranscoding;
 
   const ServerFeatures({
     required this.trash,
@@ -15,6 +16,7 @@ class ServerFeatures {
     required this.passwordLogin,
     this.ocr = false,
     this.smartSearch = false,
+    this.realtimeTranscoding = false,
   });
 
   ServerFeatures copyWith({
@@ -24,6 +26,7 @@ class ServerFeatures {
     bool? passwordLogin,
     bool? ocr,
     bool? smartSearch,
+    bool? realtimeTranscoding,
   }) {
     return ServerFeatures(
       trash: trash ?? this.trash,
@@ -32,12 +35,13 @@ class ServerFeatures {
       passwordLogin: passwordLogin ?? this.passwordLogin,
       ocr: ocr ?? this.ocr,
       smartSearch: smartSearch ?? this.smartSearch,
+      realtimeTranscoding: realtimeTranscoding ?? this.realtimeTranscoding,
     );
   }
 
   @override
   String toString() {
-    return 'ServerFeatures(trash: $trash, map: $map, oauthEnabled: $oauthEnabled, passwordLogin: $passwordLogin, ocr: $ocr, smartSearch: $smartSearch)';
+    return 'ServerFeatures(trash: $trash, map: $map, oauthEnabled: $oauthEnabled, passwordLogin: $passwordLogin, ocr: $ocr, smartSearch: $smartSearch, realtimeTranscoding: $realtimeTranscoding)';
   }
 
   ServerFeatures.fromDto(ServerFeaturesDto dto)
@@ -46,7 +50,8 @@ class ServerFeatures {
       oauthEnabled = dto.oauth,
       passwordLogin = dto.passwordLogin,
       ocr = dto.ocr,
-      smartSearch = dto.smartSearch;
+      smartSearch = dto.smartSearch,
+      realtimeTranscoding = dto.realtimeTranscoding;
 
   @override
   bool operator ==(covariant ServerFeatures other) {
@@ -59,7 +64,8 @@ class ServerFeatures {
         other.oauthEnabled == oauthEnabled &&
         other.passwordLogin == passwordLogin &&
         other.ocr == ocr &&
-        other.smartSearch == smartSearch;
+        other.smartSearch == smartSearch &&
+        other.realtimeTranscoding == realtimeTranscoding;
   }
 
   @override
@@ -69,6 +75,7 @@ class ServerFeatures {
         oauthEnabled.hashCode ^
         passwordLogin.hashCode ^
         ocr.hashCode ^
-        smartSearch.hashCode;
+        smartSearch.hashCode ^
+        realtimeTranscoding.hashCode;
   }
 }

mobile/lib/presentation/pages/drift_slideshow.page.dart

@@ -377,7 +377,7 @@ class _DriftSlideshowPageState extends ConsumerState<DriftSlideshowPage> with Si
         child: NativeVideoViewer(
           asset: asset,
           isCurrent: isCurrent,
-          image: Image(image: imageProvider, fit: BoxFit.contain, alignment: Alignment.center),
+          imageProvider: imageProvider,
         ),
       );
     }

mobile/lib/presentation/widgets/asset_viewer/asset_page.widget.dart

@@ -388,7 +388,7 @@ class _AssetPageState extends ConsumerState<AssetPage> {
         asset: asset,
         localFilePath: localFilePath,
         isCurrent: isCurrent,
-        image: Image(image: imageProvider, fit: BoxFit.contain, alignment: Alignment.center),
+        imageProvider: imageProvider,
       ),
     );
   }

mobile/lib/presentation/widgets/asset_viewer/video_viewer.widget.dart

@@ -2,6 +2,7 @@ import 'dart:async';
 import 'dart:io';
 
 import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/domain/models/store.model.dart';
@@ -14,22 +15,31 @@ import 'package:immich_mobile/providers/asset_viewer/video_player_provider.dart'
 import 'package:immich_mobile/providers/cast.provider.dart';
 import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
 import 'package:immich_mobile/providers/infrastructure/settings.provider.dart';
+import 'package:immich_mobile/providers/server_info.provider.dart';
 import 'package:immich_mobile/services/api.service.dart';
 import 'package:logging/logging.dart';
 import 'package:native_video_player/native_video_player.dart';
 
+final _hlsVideoSessionIdRegex = RegExp(
+  r'/video/stream/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/',
+);
+// For BC if we add an audio endpoint
+final _hlsAudioSessionIdRegex = RegExp(
+  r'/audio/stream/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/',
+);
+
 class NativeVideoViewer extends ConsumerStatefulWidget {
   final BaseAsset asset;
   final String? localFilePath;
   final bool isCurrent;
   final bool showControls;
-  final Widget image;
+  final ImageProvider imageProvider;
 
   const NativeVideoViewer({
     super.key,
     required this.asset,
     this.localFilePath,
-    required this.image,
+    required this.imageProvider,
     this.isCurrent = false,
     this.showControls = true,
   });
@@ -46,6 +56,7 @@ class _NativeVideoViewerState extends ConsumerState<NativeVideoViewer> with Widg
   Timer? _loadTimer;
   bool _isVideoReady = false;
   bool _shouldPlayOnForeground = true;
+  String? _remoteAssetId;
 
   VideoPlayerNotifier get _notifier => ref.read(videoPlayerProvider(widget.asset.heroTag).notifier);
 
@@ -67,11 +78,16 @@ class _NativeVideoViewerState extends ConsumerState<NativeVideoViewer> with Widg
     if (!widget.isCurrent) {
       _loadTimer?.cancel();
       _notifier.pause();
+      _notifier.endHlsSession();
       return;
     }
 
-    // Prevent unnecessary loading when swiping between assets.
-    _loadTimer = Timer(const Duration(milliseconds: 200), _loadVideo);
+    if (ref.read(serverInfoProvider).serverFeatures.realtimeTranscoding) {
+      _loadVideo();
+    } else {
+      // Prevent unnecessary loading when swiping between assets.
+      _loadTimer = Timer(const Duration(milliseconds: 200), _loadVideo);
+    }
   }
 
   @override
@@ -141,14 +157,22 @@ class _NativeVideoViewerState extends ConsumerState<NativeVideoViewer> with Widg
         );
       }
 
-      final remoteId = (videoAsset as RemoteAsset).id;
-
-      final serverEndpoint = Store.get(StoreKey.serverEndpoint);
       final isOriginalVideo = ref.read(appConfigProvider).viewer.loadOriginalVideo;
-      final String postfixUrl = isOriginalVideo ? 'original' : 'video/playback';
-      final String videoUrl = videoAsset.livePhotoVideoId != null
-          ? '$serverEndpoint/assets/${videoAsset.livePhotoVideoId}/$postfixUrl'
-          : '$serverEndpoint/assets/$remoteId/$postfixUrl';
+      final realtimeTranscoding = ref.read(serverInfoProvider).serverFeatures.realtimeTranscoding;
+      // Motion photo clips are short, so spinning up a transcoding session for them is wasteful
+      final useHls = !isOriginalVideo && videoAsset.livePhotoVideoId == null && realtimeTranscoding;
+      final remoteId = (videoAsset as RemoteAsset).id;
+      final serverEndpoint = Store.get(StoreKey.serverEndpoint);
+      final String videoUrl;
+      if (useHls) {
+        videoUrl = '$serverEndpoint/assets/$remoteId/video/stream/main.m3u8';
+      } else {
+        final String postfixUrl = isOriginalVideo ? 'original' : 'video/playback';
+        videoUrl = videoAsset.livePhotoVideoId != null
+            ? '$serverEndpoint/assets/${videoAsset.livePhotoVideoId}/$postfixUrl'
+            : '$serverEndpoint/assets/$remoteId/$postfixUrl';
+      }
+      _remoteAssetId = remoteId;
 
       return VideoSource.init(path: videoUrl, type: VideoSourceType.network, headers: ApiService.getRequestHeaders());
     } catch (error) {
@@ -209,11 +233,17 @@ class _NativeVideoViewerState extends ConsumerState<NativeVideoViewer> with Widg
     _notifier.onNativeStatusChanged();
   }
 
+  void _onSourceResolved() {
+    final url = _controller?.onPlaybackSourceResolved.value;
+    _notifier.updateHlsSession(assetId: _remoteAssetId, sessionId: url == null ? null : _extractHlsSessionId(url));
+  }
+
   void _removeListeners() {
     _controller?.onPlaybackPositionChanged.removeListener(_onPlaybackPositionChanged);
     _controller?.onPlaybackStatusChanged.removeListener(_onPlaybackStatusChanged);
     _controller?.onPlaybackReady.removeListener(_onPlaybackReady);
     _controller?.onPlaybackEnded.removeListener(_onPlaybackEnded);
+    _controller?.onPlaybackSourceResolved.removeListener(_onSourceResolved);
   }
 
   void _loadVideo() async {
@@ -244,6 +274,7 @@ class _NativeVideoViewerState extends ConsumerState<NativeVideoViewer> with Widg
     nc.onPlaybackStatusChanged.addListener(_onPlaybackStatusChanged);
     nc.onPlaybackReady.addListener(_onPlaybackReady);
     nc.onPlaybackEnded.addListener(_onPlaybackEnded);
+    nc.onPlaybackSourceResolved.addListener(_onSourceResolved);
 
     _controller = nc;
 
@@ -252,30 +283,85 @@ class _NativeVideoViewerState extends ConsumerState<NativeVideoViewer> with Widg
     }
   }
 
+  /// Extracts the HLS session id from a resolved playlist or segment URL,
+  /// e.g. `https://host/api/assets/{id}/video/stream/{sessionId}/0/playlist.m3u8`.
+  String? _extractHlsSessionId(String url) =>
+      _hlsVideoSessionIdRegex.firstMatch(url)?.group(1) ?? _hlsAudioSessionIdRegex.firstMatch(url)?.group(1);
+
   @override
   Widget build(BuildContext context) {
-    final isCasting = ref.watch(castProvider.select((c) => c.isCasting));
-    final status = ref.watch(videoPlayerProvider(widget.asset.heroTag).select((v) => v.status));
+    final image = Image(image: widget.imageProvider, fit: BoxFit.contain, alignment: Alignment.center);
+    if (ref.watch(castProvider.select((c) => c.isCasting))) {
+      return IgnorePointer(child: Center(child: image));
+    }
 
+    final status = ref.watch(videoPlayerProvider(widget.asset.heroTag).select((v) => v.status));
     return IgnorePointer(
       child: Stack(
+        clipBehavior: Clip.none,
         children: [
-          Center(child: widget.image),
-          if (!isCasting) ...[
-            Visibility.maintain(
-              visible: _isVideoReady,
-              child: NativeVideoPlayerView(onViewReady: _initController),
-            ),
-            Center(
-              child: AnimatedOpacity(
-                opacity: status == VideoPlaybackStatus.buffering ? 1.0 : 0.0,
-                duration: const Duration(milliseconds: 400),
-                child: const CircularProgressIndicator(),
-              ),
-            ),
-          ],
+          // The engine snaps the video platform view to the device-pixel grid; snap the
+          // placeholder the same way so it doesn't show a hairline past the video's edge.
+          _DevicePixelSnap(devicePixelRatio: MediaQuery.devicePixelRatioOf(context), child: image),
+          Visibility.maintain(
+            visible: _isVideoReady,
+            child: NativeVideoPlayerView(onViewReady: _initController),
+          ),
+          if (status == VideoPlaybackStatus.buffering) const Center(child: CircularProgressIndicator()),
         ],
       ),
     );
   }
 }
+
+class _DevicePixelSnap extends SingleChildRenderObjectWidget {
+  final double devicePixelRatio;
+
+  const _DevicePixelSnap({required this.devicePixelRatio, required Widget super.child});
+
+  @override
+  _RenderDevicePixelSnap createRenderObject(BuildContext context) => _RenderDevicePixelSnap(devicePixelRatio);
+
+  @override
+  void updateRenderObject(BuildContext context, _RenderDevicePixelSnap renderObject) {
+    renderObject.devicePixelRatio = devicePixelRatio;
+  }
+}
+
+class _RenderDevicePixelSnap extends RenderShiftedBox {
+  _RenderDevicePixelSnap(this._devicePixelRatio) : super(null);
+
+  double _devicePixelRatio;
+  set devicePixelRatio(double value) {
+    if (_devicePixelRatio == value) {
+      return;
+    }
+    _devicePixelRatio = value;
+    markNeedsLayout();
+  }
+
+  /// The largest device-pixel-aligned extent that still tucks under the platform view.
+  double _snap(double extent) {
+    final scaled = extent * _devicePixelRatio;
+    final floored = scaled.floorToDouble();
+    if (floored == scaled) {
+      return extent;
+    }
+    final pixels = floored - 1;
+    return pixels <= 0 ? extent : pixels / _devicePixelRatio;
+  }
+
+  @override
+  Size computeDryLayout(BoxConstraints constraints) => constraints.biggest;
+
+  @override
+  void performLayout() {
+    size = constraints.biggest;
+    final child = this.child;
+    if (child == null) {
+      return;
+    }
+    child.layout(BoxConstraints.tight(Size(_snap(size.width), _snap(size.height))));
+    (child.parentData! as BoxParentData).offset = Offset.zero;
+  }
+}

mobile/lib/presentation/widgets/memory/memory_card.widget.dart

@@ -66,7 +66,7 @@ class DriftMemoryCard extends StatelessWidget {
                     asset: asset,
                     isCurrent: isCurrent,
                     showControls: false,
-                    image: FullImage(asset, size: context.sizeData, fit: BoxFit.contain),
+                    imageProvider: getFullImageProvider(asset, size: context.sizeData),
                   ),
                 ),
               );

mobile/lib/providers/asset_viewer/video_player_provider.dart

@@ -1,8 +1,10 @@
 import 'dart:async';
 
 import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/providers/api.provider.dart';
 import 'package:logging/logging.dart';
 import 'package:native_video_player/native_video_player.dart';
+import 'package:openapi/api.dart';
 import 'package:wakelock_plus/wakelock_plus.dart';
 
 enum VideoPlaybackStatus { paused, playing, buffering, completed }
@@ -33,21 +35,25 @@ final videoPlayerProvider = StateNotifierProvider.autoDispose.family<VideoPlayer
   ref,
   name,
 ) {
-  return VideoPlayerNotifier();
+  return VideoPlayerNotifier(ref.read(apiServiceProvider).assetsApi);
 });
 
 class VideoPlayerNotifier extends StateNotifier<VideoPlayerState> {
   static final _log = Logger('VideoPlayerNotifier');
 
-  VideoPlayerNotifier() : super(_defaultState);
+  VideoPlayerNotifier(this._assetsApi) : super(_defaultState);
 
+  final AssetsApi _assetsApi;
   NativeVideoPlayerController? _controller;
   Timer? _bufferingTimer;
   Timer? _seekTimer;
   VideoPlaybackStatus? _holdStatus;
+  String? _hlsAssetId;
+  String? _hlsSessionId;
 
   @override
   void dispose() {
+    endHlsSession();
     _bufferingTimer?.cancel();
     _seekTimer?.cancel();
     WakelockPlus.disable();
@@ -60,6 +66,29 @@ class VideoPlayerNotifier extends StateNotifier<VideoPlayerState> {
     _controller = controller;
   }
 
+  void updateHlsSession({required String? assetId, required String? sessionId}) {
+    if (sessionId == null || sessionId == _hlsSessionId) {
+      return;
+    }
+    endHlsSession();
+    _hlsAssetId = assetId;
+    _hlsSessionId = sessionId;
+  }
+
+  void endHlsSession() {
+    final assetId = _hlsAssetId;
+    final sessionId = _hlsSessionId;
+    _hlsSessionId = null;
+    if (assetId == null || sessionId == null) {
+      return;
+    }
+    unawaited(
+      _assetsApi.endSession(assetId, sessionId).onError((error, stackTrace) {
+        _log.warning('Failed to end HLS session $sessionId for asset $assetId', error, stackTrace);
+      }),
+    );
+  }
+
   Future<void> load(VideoSource source) async {
     _startBufferingTimer();
     try {

mobile/openapi/lib/api/assets_api.dart

@@ -982,7 +982,9 @@ class AssetsApi {
   /// * [String] key:
   ///
   /// * [String] slug:
-  Future<Response> getMediaPlaylistWithHttpInfo(String id, String sessionId, int variantIndex, { String? key, String? slug, Future<void>? abortTrigger, }) async {
+  ///
+  /// * [num] xImmichHlsPos:
+  Future<Response> getMediaPlaylistWithHttpInfo(String id, String sessionId, int variantIndex, { String? key, String? slug, num? xImmichHlsPos, Future<void>? abortTrigger, }) async {
     // ignore: prefer_const_declarations
     final apiPath = r'/assets/{id}/video/stream/{sessionId}/{variantIndex}/playlist.m3u8'
       .replaceAll('{id}', id)
@@ -1003,6 +1005,10 @@ class AssetsApi {
       queryParams.addAll(_queryParams('', 'slug', slug));
     }
 
+    if (xImmichHlsPos != null) {
+      headerParams[r'x-immich-hls-pos'] = parameterToString(xImmichHlsPos);
+    }
+
     const contentTypes = <String>[];
 
 
@@ -1033,8 +1039,10 @@ class AssetsApi {
   /// * [String] key:
   ///
   /// * [String] slug:
-  Future<String?> getMediaPlaylist(String id, String sessionId, int variantIndex, { String? key, String? slug, Future<void>? abortTrigger, }) async {
-    final response = await getMediaPlaylistWithHttpInfo(id, sessionId, variantIndex, key: key, slug: slug, abortTrigger: abortTrigger,);
+  ///
+  /// * [num] xImmichHlsPos:
+  Future<String?> getMediaPlaylist(String id, String sessionId, int variantIndex, { String? key, String? slug, num? xImmichHlsPos, Future<void>? abortTrigger, }) async {
+    final response = await getMediaPlaylistWithHttpInfo(id, sessionId, variantIndex, key: key, slug: slug, xImmichHlsPos: xImmichHlsPos, abortTrigger: abortTrigger,);
     if (response.statusCode >= HttpStatus.badRequest) {
       throw ApiException(response.statusCode, await _decodeBodyBytes(response));
     }

mobile/pubspec.lock

@@ -1128,8 +1128,8 @@ packages:
     dependency: "direct main"
     description:
       path: "."
-      ref: cdf621bdb7edaf996e118a58a48f6441187d79c6
-      resolved-ref: cdf621bdb7edaf996e118a58a48f6441187d79c6
+      ref: "13d3c6bc971281ee5e2e5f677f0dfa9da8fcc934"
+      resolved-ref: "13d3c6bc971281ee5e2e5f677f0dfa9da8fcc934"
       url: "https://github.com/immich-app/native_video_player"
     source: git
     version: "1.3.1"

mobile/pubspec.yaml

@@ -48,7 +48,7 @@ dependencies:
   native_video_player:
     git:
       url: https://github.com/immich-app/native_video_player
-      ref: 'cdf621bdb7edaf996e118a58a48f6441187d79c6'
+      ref: '13d3c6bc971281ee5e2e5f677f0dfa9da8fcc934'
   network_info_plus: ^6.1.4
   octo_image: ^2.1.0
   openapi:

open-api/immich-openapi-specs.json

@@ -4924,6 +4924,15 @@
               "maximum": 9007199254740991,
               "type": "integer"
             }
+          },
+          {
+            "name": "x-immich-hls-pos",
+            "required": false,
+            "in": "header",
+            "schema": {
+              "minimum": 0,
+              "type": "number"
+            }
           }
         ],
         "responses": {

packages/sdk/src/fetch-client.ts

@@ -4452,12 +4452,13 @@ export function endSession({ id, key, sessionId, slug }: {
 /**
  * Get HLS media playlist
  */
-export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: {
+export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex, xImmichHlsPos }: {
     id: string;
     key?: string;
     sessionId: string;
     slug?: string;
     variantIndex: number;
+    xImmichHlsPos?: number;
 }, opts?: Oazapfts.RequestOpts) {
     return oazapfts.ok(oazapfts.fetchBlob<{
         status: 200;
@@ -4466,7 +4467,10 @@ export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: {
         key,
         slug
     }))}`, {
-        ...opts
+        ...opts,
+        headers: oazapfts.mergeHeaders(opts?.headers, {
+            "x-immich-hls-pos": xImmichHlsPos
+        })
     }));
 }
 /**

server/src/constants.ts

@@ -223,6 +223,12 @@ export const SUPPORTED_HWA_CODECS: Record<TranscodeHardwareAcceleration, VideoCo
 export const HLS_BACKPRESSURE_PAUSE_SEGMENTS = 30;
 export const HLS_BACKPRESSURE_RESUME_SEGMENTS = 15;
 export const HLS_CLEANUP_INTERVAL_MS = 60 * 1000;
+export const HLS_CRF: Record<VideoCodec, number> = {
+  [VideoCodec.H264]: 23,
+  [VideoCodec.Hevc]: 28,
+  [VideoCodec.Vp9]: 31,
+  [VideoCodec.Av1]: 35,
+};
 export const HLS_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000;
 export const HLS_LEASE_DURATION_MS = 30 * 60 * 1000;
 export const HLS_PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl';

server/src/controllers/video-stream.controller.ts

@@ -6,6 +6,7 @@ import { HLS_PLAYLIST_CONTENT_TYPE } from 'src/constants';
 import { Endpoint, HistoryBuilder } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
 import {
+  HlsPlaylistHeaderDto,
   HlsSegmentHeaderDto,
   HlsSegmentParamDto,
   HlsSessionParamDto,
@@ -50,8 +51,17 @@ export class VideoStreamController {
     description: 'Returns an HLS media playlist for one variant of the streaming session.',
     history: new HistoryBuilder().added('v3').alpha('v3'),
   })
-  getMediaPlaylist(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsVariantParamDto) {
-    return this.service.getMediaPlaylist(auth, id, sessionId);
+  getMediaPlaylist(
+    @Auth() auth: AuthDto,
+    @Param() { id, sessionId, variantIndex }: HlsVariantParamDto,
+    @Headers() headers: HlsPlaylistHeaderDto,
+  ) {
+    try {
+      headers = HlsPlaylistHeaderDto.create(headers);
+    } catch (error) {
+      throw new ZodValidationException(error);
+    }
+    return this.service.getMediaPlaylist(auth, id, sessionId, variantIndex, headers[ImmichHeader.HlsPosition]);
   }
 
   @Get(':id/video/stream/:sessionId/:variantIndex/:filename')

server/src/dtos/streaming.dto.ts

@@ -32,3 +32,11 @@ const HlsSegmentHeaderSchema = z.object({
 });
 
 export class HlsSegmentHeaderDto extends createZodDto(HlsSegmentHeaderSchema) {}
+
+const HlsPlaylistHeaderSchema = z.object({
+  // Lets the client hint at which segment will be loaded after the playlist.
+  // A position rather than a segment index since indices aren't comparable across variants.
+  [ImmichHeader.HlsPosition]: z.coerce.number().min(0).optional(),
+});
+
+export class HlsPlaylistHeaderDto extends createZodDto(HlsPlaylistHeaderSchema) {}

server/src/enum.ts

@@ -25,6 +25,7 @@ export enum ImmichHeader {
   Checksum = 'x-immich-checksum',
   CorrelationId = 'X-Correlation-ID',
   HlsInitSegment = 'x-immich-hls-msn',
+  HlsPosition = 'x-immich-hls-pos',
 }
 
 export enum ImmichQuery {

server/src/services/hls.service.spec.ts

@@ -8,6 +8,7 @@ import { newTestService, ServiceMocks } from 'test/utils';
 // EXTINF values come from FFmpeg's playlist to enforce an exact match
 const eiffelExpectedMediaPlaylist = `#EXTM3U
 #EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
 #EXT-X-TARGETDURATION:2
 #EXT-X-MEDIA-SEQUENCE:0
 #EXT-X-PLAYLIST-TYPE:VOD
@@ -41,6 +42,7 @@ seg_11.m4s
 
 const waterfallExpectedMediaPlaylist = `#EXTM3U
 #EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
 #EXT-X-TARGETDURATION:2
 #EXT-X-MEDIA-SEQUENCE:0
 #EXT-X-PLAYLIST-TYPE:VOD
@@ -62,6 +64,7 @@ seg_5.m4s
 
 const trainExpectedMediaPlaylist = `#EXTM3U
 #EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
 #EXT-X-TARGETDURATION:2
 #EXT-X-MEDIA-SEQUENCE:0
 #EXT-X-PLAYLIST-TYPE:VOD
@@ -95,6 +98,7 @@ const sessionId = '00000000-0000-0000-0000-000000000000';
 
 const eiffelExpectedMasterDisabled = `#EXTM3U
 #EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
 #EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
 ${sessionId}/0/playlist.m3u8
 #EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
@@ -117,6 +121,7 @@ ${sessionId}/8/playlist.m3u8
 
 const eiffelExpectedMasterRkmpp = `#EXTM3U
 #EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
 #EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
 ${sessionId}/1/playlist.m3u8
 #EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
@@ -133,6 +138,7 @@ ${sessionId}/8/playlist.m3u8
 
 const waterfallExpectedMasterDisabled = `#EXTM3U
 #EXT-X-VERSION:7
+#EXT-X-INDEPENDENT-SEGMENTS
 #EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
 ${sessionId}/0/playlist.m3u8
 #EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
@@ -218,12 +224,58 @@ describe(HlsService.name, () => {
     it.each(fixtures)('matches FFmpeg for $data.originalPath', async ({ data, playlist }) => {
       mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
       mocks.videoStream.getForMediaPlaylist.mockResolvedValue(data);
-      await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).resolves.toBe(playlist);
+      await expect(sut.getMediaPlaylist(auth, assetId, sessionId, 0)).resolves.toBe(playlist);
     });
 
     it('throws NotFoundException when the session/asset cannot be loaded', async () => {
       mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
-      await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).rejects.toBeInstanceOf(NotFoundException);
+      await expect(sut.getMediaPlaylist(auth, assetId, sessionId, 0)).rejects.toBeInstanceOf(NotFoundException);
+    });
+
+    it('prewarms transcoding at the segment containing the hinted position', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);
+      await sut.getMediaPlaylist(auth, assetId, sessionId, 1, 10.5);
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSegmentRequest', {
+        sessionId,
+        assetId,
+        variantIndex: 1,
+        segmentIndex: 5,
+      });
+    });
+
+    it('prewarms from the last requested segment when no hint is given', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never);
+      mocks.storage.checkFileExists.mockResolvedValue(true);
+      await sut.getSegment(auth, assetId, sessionId, 0, 'seg_5.m4s');
+
+      mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);
+      await sut.getMediaPlaylist(auth, assetId, sessionId, 1);
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSegmentRequest', {
+        sessionId,
+        assetId,
+        variantIndex: 1,
+        segmentIndex: 6,
+      });
+    });
+
+    it('does not prewarm without a hint or prior segment traffic', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);
+      await sut.getMediaPlaylist(auth, assetId, sessionId, 1);
+      expect(mocks.websocket.serverSend).not.toHaveBeenCalled();
+    });
+
+    it('does not prewarm the variant the session is already playing', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never);
+      mocks.storage.checkFileExists.mockResolvedValue(true);
+      await sut.getSegment(auth, assetId, sessionId, 1, 'seg_5.m4s');
+
+      mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);
+      await sut.getMediaPlaylist(auth, assetId, sessionId, 1, 12.5);
+      expect(mocks.websocket.serverSend).not.toHaveBeenCalledWith('HlsSegmentRequest', expect.anything());
     });
   });
 
@@ -314,7 +366,7 @@ describe(HlsService.name, () => {
       );
     });
 
-    it('uses the target segment for init.mp4 when provided', async () => {
+    it('uses the initSegment hint for init.mp4', async () => {
       await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 7);
       expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
         sessionId,
@@ -323,18 +375,18 @@ describe(HlsService.name, () => {
       });
     });
 
-    it('prefers the target segment over the lastRequested + 1 fallback', async () => {
+    it('prefers the initSegment hint over the lastRequested + 1 fallback', async () => {
       await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); // fallback would be 6
       mocks.websocket.serverSend.mockClear();
-      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 12);
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 10);
       expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
         sessionId,
         variantIndex,
-        segmentIndex: 12,
+        segmentIndex: 10,
       });
     });
 
-    it('ignores the target segment for media segment requests (the filename wins)', async () => {
+    it('ignores the initSegment hint for media segment requests (the filename wins)', async () => {
       await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s', 99);
       expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
         sessionId,

server/src/services/hls.service.ts

@@ -21,6 +21,7 @@ import { ImmichFileResponse } from 'src/utils/file';
 import { getOutputSize } from 'src/utils/media';
 
 type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo };
+type Segmentation = { fps: number; framesPerSegment: number; segmentCount: number; segmentDuration: number };
 type ApiSession = { lastRequestedSegment: number | null; lastVariantIndex: number | null };
 
 @Injectable()
@@ -71,7 +72,7 @@ export class HlsService extends BaseService {
     return this.generateMainPlaylist(sessionId, ffmpeg, asset);
   }
 
-  async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string) {
+  async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, position?: number) {
     await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
 
     const asset = await this.videoStreamRepository.getForMediaPlaylist(assetId, sessionId);
@@ -79,7 +80,11 @@ export class HlsService extends BaseService {
       throw new NotFoundException('Asset not found or metadata not yet ready for streaming');
     }
 
-    return this.generateMediaPlaylist(asset);
+    const segmentation = this.getSegmentation(asset);
+    const hintedSegment = position === undefined ? undefined : this.positionToSegmentIndex(segmentation, position);
+    this.prewarmVariant(assetId, sessionId, variantIndex, hintedSegment);
+
+    return this.generateMediaPlaylist(asset, segmentation);
   }
 
   async getSegment(
@@ -129,7 +134,7 @@ export class HlsService extends BaseService {
     const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3);
     const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width);
     const targetResolution = Math.max(sourceResolution, HLS_VARIANTS[0].resolution);
-    const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`];
+    const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`, '#EXT-X-INDEPENDENT-SEGMENTS'];
     for (let i = 0; i < HLS_VARIANTS.length; i++) {
       const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i];
       if (resolution > targetResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) {
@@ -143,24 +148,33 @@ export class HlsService extends BaseService {
     }
     lines.push('');
 
-    if (lines.length === 3) {
+    if (lines.length === 4) {
       throw new NotFoundException('No supported variants for this video');
     }
 
     return lines.join('\n');
   }
 
-  private generateMediaPlaylist({ videoStream, packets }: AssetWithStreamInfo) {
+  private getSegmentation({ videoStream, packets }: AssetWithStreamInfo): Segmentation {
     const fps = (packets.packetCount * videoStream.timeBase) / packets.totalDuration;
     const framesPerSegment = Math.ceil(HLS_SEGMENT_DURATION * fps);
-    const fullSegmentDuration = framesPerSegment / fps;
     const segmentCount = Math.ceil(packets.outputFrames / framesPerSegment);
+    return { fps, framesPerSegment, segmentCount, segmentDuration: framesPerSegment / fps };
+  }
+
+  private positionToSegmentIndex({ segmentDuration, segmentCount }: Segmentation, position: number) {
+    return Math.min(Math.max(Math.floor(position / segmentDuration), 0), segmentCount - 1);
+  }
+
+  private generateMediaPlaylist({ packets }: AssetWithStreamInfo, segmentation: Segmentation) {
+    const { fps, framesPerSegment, segmentCount, segmentDuration: fullSegmentDuration } = segmentation;
     const lastSegmentFrames = packets.outputFrames - framesPerSegment * (segmentCount - 1);
     const lastSegmentDuration = lastSegmentFrames / fps;
 
     const lines = [
       '#EXTM3U',
       `#EXT-X-VERSION:${HLS_VERSION}`,
+      '#EXT-X-INDEPENDENT-SEGMENTS',
       `#EXT-X-TARGETDURATION:${HLS_SEGMENT_DURATION}`,
       '#EXT-X-MEDIA-SEQUENCE:0',
       '#EXT-X-PLAYLIST-TYPE:VOD',
@@ -175,6 +189,19 @@ export class HlsService extends BaseService {
     return lines.join('\n');
   }
 
+  private prewarmVariant(assetId: string, sessionId: string, variantIndex: number, hintedSegment?: number) {
+    const session = this.sessions.get(sessionId);
+    if (session?.lastVariantIndex === variantIndex) {
+      return;
+    }
+
+    const nextSegment = session && session.lastRequestedSegment !== null ? session.lastRequestedSegment + 1 : undefined;
+    const segmentIndex = hintedSegment ?? nextSegment;
+    if (segmentIndex !== undefined) {
+      this.websocketRepository.serverSend('HlsSegmentRequest', { sessionId, assetId, variantIndex, segmentIndex });
+    }
+  }
+
   private getSegmentKey({ sessionId, variantIndex, segmentIndex }: ArgOf<'HlsSegmentResult'>) {
     return `${sessionId}:${variantIndex}:${segmentIndex}`;
   }

server/src/services/transcoding.service.ts

@@ -5,6 +5,7 @@ import {
   HLS_BACKPRESSURE_PAUSE_SEGMENTS,
   HLS_BACKPRESSURE_RESUME_SEGMENTS,
   HLS_CLEANUP_INTERVAL_MS,
+  HLS_CRF,
   HLS_INACTIVITY_TIMEOUT_MS,
   HLS_LEASE_DURATION_MS,
   HLS_SEGMENT_DURATION,
@@ -221,6 +222,7 @@ export class TranscodingService extends BaseService {
           targetResolution: String(variant.resolution),
           maxBitrate: `${Math.round(variant.bitrate / 1000)}k`,
           gopSize: gop,
+          crf: HLS_CRF[variant.codec],
         },
         this.videoInterfaces,
         { strictGop: true, lowLatency: true },