unify hls playback

minor simplifications

capLevelToPlayerSize

custom media element

minor tweaks

respect playOriginal

cleanup

mobile/openapi/lib/api/assets_api.dart

@@ -1067,7 +1067,9 @@ class AssetsApi {
   /// * [String] key:
   ///
   /// * [String] slug:
-  Future<Response> getSegmentWithHttpInfo(String filename, String id, String sessionId, int variantIndex, { String? key, String? slug, Future<void>? abortTrigger, }) async {
+  ///
+  /// * [int] xImmichHlsMsn:
+  Future<Response> getSegmentWithHttpInfo(String filename, String id, String sessionId, int variantIndex, { String? key, String? slug, int? xImmichHlsMsn, Future<void>? abortTrigger, }) async {
     // ignore: prefer_const_declarations
     final apiPath = r'/assets/{id}/video/stream/{sessionId}/{variantIndex}/{filename}'
       .replaceAll('{filename}', filename)
@@ -1089,6 +1091,10 @@ class AssetsApi {
       queryParams.addAll(_queryParams('', 'slug', slug));
     }
 
+    if (xImmichHlsMsn != null) {
+      headerParams[r'x-immich-hls-msn'] = parameterToString(xImmichHlsMsn);
+    }
+
     const contentTypes = <String>[];
 
 
@@ -1121,8 +1127,10 @@ class AssetsApi {
   /// * [String] key:
   ///
   /// * [String] slug:
-  Future<MultipartFile?> getSegment(String filename, String id, String sessionId, int variantIndex, { String? key, String? slug, Future<void>? abortTrigger, }) async {
-    final response = await getSegmentWithHttpInfo(filename, id, sessionId, variantIndex, key: key, slug: slug, abortTrigger: abortTrigger,);
+  ///
+  /// * [int] xImmichHlsMsn:
+  Future<MultipartFile?> getSegment(String filename, String id, String sessionId, int variantIndex, { String? key, String? slug, int? xImmichHlsMsn, Future<void>? abortTrigger, }) async {
+    final response = await getSegmentWithHttpInfo(filename, id, sessionId, variantIndex, key: key, slug: slug, xImmichHlsMsn: xImmichHlsMsn, abortTrigger: abortTrigger,);
     if (response.statusCode >= HttpStatus.badRequest) {
       throw ApiException(response.statusCode, await _decodeBodyBytes(response));
     }

open-api/immich-openapi-specs.json

@@ -4734,6 +4734,16 @@
               "maximum": 9007199254740991,
               "type": "integer"
             }
+          },
+          {
+            "name": "x-immich-hls-msn",
+            "required": false,
+            "in": "header",
+            "schema": {
+              "minimum": 0,
+              "maximum": 9007199254740991,
+              "type": "integer"
+            }
           }
         ],
         "responses": {

packages/sdk/src/fetch-client.ts

@@ -4366,13 +4366,14 @@ export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: {
 /**
  * Get HLS segment or init file
  */
-export function getSegment({ filename, id, key, sessionId, slug, variantIndex }: {
+export function getSegment({ filename, id, key, sessionId, slug, variantIndex, xImmichHlsMsn }: {
     filename: string;
     id: string;
     key?: string;
     sessionId: string;
     slug?: string;
     variantIndex: number;
+    xImmichHlsMsn?: number;
 }, opts?: Oazapfts.RequestOpts) {
     return oazapfts.ok(oazapfts.fetchBlob<{
         status: 200;
@@ -4381,7 +4382,10 @@ export function getSegment({ filename, id, key, sessionId, slug, variantIndex }:
         key,
         slug
     }))}`, {
-        ...opts
+        ...opts,
+        headers: oazapfts.mergeHeaders(opts?.headers, {
+            "x-immich-hls-msn": xImmichHlsMsn
+        })
     }));
 }
 /**

pnpm-lock.yaml

@@ -801,6 +801,9 @@ importers:
       '@zoom-image/svelte':
         specifier: ^0.3.0
         version: 0.3.9(svelte@5.55.8(@typescript-eslint/types@8.59.4))
+      custom-media-element:
+        specifier: ^1.4.6
+        version: 1.4.6
       dom-to-image:
         specifier: ^2.6.0
         version: 2.6.0
@@ -819,12 +822,9 @@ importers:
       happy-dom:
         specifier: ^20.0.0
         version: 20.9.0
-      hls-video-element:
-        specifier: ^1.5.11
-        version: 1.5.11
       hls.js:
-        specifier: ^1.6.16
-        version: 1.6.16
+        specifier: 1.7.0-beta.1.0.canary.11837
+        version: 1.7.0-beta.1.0.canary.11837
       intl-messageformat:
         specifier: ^11.0.0
         version: 11.2.6
@@ -8258,11 +8258,8 @@ packages:
   history@4.10.1:
     resolution: {integrity: sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==}
 
-  hls-video-element@1.5.11:
-    resolution: {integrity: sha512-tJJ65/52CDxj8XFyIve6zT9nVVdUIc6mqvKR25X0ycPKHk07rpjp4xxVteeCefDUBSf/tFLhlICFmn3KWj37xA==}
-
-  hls.js@1.6.16:
-    resolution: {integrity: sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==}
+  hls.js@1.7.0-beta.1.0.canary.11837:
+    resolution: {integrity: sha512-JXSTYLvxCpJeD8xgYlIYzEA0Ag+1Vnkakl7y8JiS0RtgNPFgtsGjqqxCPFyYCmumi1CON6VtksohqeJoiBAKmw==}
 
   hogan.js@3.0.2:
     resolution: {integrity: sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==}
@@ -9261,9 +9258,6 @@ packages:
   media-chrome@4.19.0:
     resolution: {integrity: sha512-HWhDTwts+BSbdPkkB1VsJXp5kvL0IxY7xFT5tBwliM2+89kTPVTnHnev+9it2f9PweANjT/C8/C/S0PW9oyZbA==}
 
-  media-tracks@0.3.5:
-    resolution: {integrity: sha512-l54rkKXlLBt3ob3zOLWHcnjvwUmX5bNEZ70igyapOZZC9imzqBmq1oz8p2roiV04KhjblFIi2hetLPF1oYVLRA==}
-
   media-typer@0.3.0:
     resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
     engines: {node: '>= 0.6'}
@@ -21528,13 +21522,7 @@ snapshots:
       tiny-warning: 1.0.3
       value-equal: 1.0.1
 
-  hls-video-element@1.5.11:
-    dependencies:
-      custom-media-element: 1.4.6
-      hls.js: 1.6.16
-      media-tracks: 0.3.5
-
-  hls.js@1.6.16: {}
+  hls.js@1.7.0-beta.1.0.canary.11837: {}
 
   hogan.js@3.0.2:
     dependencies:
@@ -22625,8 +22613,6 @@ snapshots:
     transitivePeerDependencies:
       - react
 
-  media-tracks@0.3.5: {}
-
   media-typer@0.3.0: {}
 
   media-typer@1.1.0: {}

server/src/controllers/video-stream.controller.ts

@@ -1,11 +1,17 @@
-import { Controller, Delete, Get, Header, HttpCode, HttpStatus, Next, Param, Res } from '@nestjs/common';
+import { Controller, Delete, Get, Header, Headers, HttpCode, HttpStatus, Next, Param, Res } from '@nestjs/common';
 import { ApiProduces, ApiTags } from '@nestjs/swagger';
 import { NextFunction, Response } from 'express';
+import { ZodValidationException } from 'nestjs-zod';
 import { HLS_PLAYLIST_CONTENT_TYPE } from 'src/constants';
 import { Endpoint, HistoryBuilder } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
-import { HlsSegmentParamDto, HlsSessionParamDto, HlsVariantParamDto } from 'src/dtos/streaming.dto';
-import { ApiTag, Permission, RouteKey } from 'src/enum';
+import {
+  HlsSegmentHeaderDto,
+  HlsSegmentParamDto,
+  HlsSessionParamDto,
+  HlsVariantParamDto,
+} from 'src/dtos/streaming.dto';
+import { ApiTag, ImmichHeader, Permission, RouteKey } from 'src/enum';
 import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { HlsService } from 'src/services/hls.service';
@@ -59,10 +65,21 @@ export class VideoStreamController {
   async getSegment(
     @Auth() auth: AuthDto,
     @Param() { id, sessionId, variantIndex, filename }: HlsSegmentParamDto,
+    @Headers() headers: HlsSegmentHeaderDto,
     @Res() res: Response,
     @Next() next: NextFunction,
   ) {
-    await sendFile(res, next, () => this.service.getSegment(auth, id, sessionId, variantIndex, filename), this.logger);
+    try {
+      headers = HlsSegmentHeaderDto.create(headers);
+    } catch (error) {
+      throw new ZodValidationException(error);
+    }
+    await sendFile(
+      res,
+      next,
+      () => this.service.getSegment(auth, id, sessionId, variantIndex, filename, headers[ImmichHeader.HlsInitSegment]),
+      this.logger,
+    );
   }
 
   @Delete(':id/video/stream/:sessionId')

server/src/dtos/streaming.dto.ts

@@ -1,4 +1,5 @@
 import { createZodDto } from 'nestjs-zod';
+import { ImmichHeader } from 'src/enum';
 import z from 'zod';
 
 const HlsSessionParamSchema = z.object({
@@ -24,3 +25,10 @@ const HlsSegmentParamSchema = z.object({
 });
 
 export class HlsSegmentParamDto extends createZodDto(HlsSegmentParamSchema) {}
+
+const HlsSegmentHeaderSchema = z.object({
+  // Lets the client hint at which segment will be loaded after init.mp4.
+  [ImmichHeader.HlsInitSegment]: z.coerce.number().int().min(0).optional(),
+});
+
+export class HlsSegmentHeaderDto extends createZodDto(HlsSegmentHeaderSchema) {}

server/src/enum.ts

@@ -24,6 +24,7 @@ export enum ImmichHeader {
   SharedLinkSlug = 'x-immich-share-slug',
   Checksum = 'x-immich-checksum',
   CorrelationId = 'X-Correlation-ID',
+  HlsInitSegment = 'x-immich-hls-msn',
 }
 
 export enum ImmichQuery {

server/src/services/hls.service.spec.ts

@@ -256,7 +256,7 @@ describe(HlsService.name, () => {
       });
     });
 
-    it('returns lastRequested + 1 for init.mp4 after a segment has been served', async () => {
+    it('returns lastRequested + 1 for init.mp4 without a target segment', async () => {
       await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
       mocks.websocket.serverSend.mockClear();
       await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
@@ -313,6 +313,35 @@ describe(HlsService.name, () => {
         NotFoundException,
       );
     });
+
+    it('uses the target segment for init.mp4 when provided', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 7);
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 7,
+      });
+    });
+
+    it('prefers the target segment over the lastRequested + 1 fallback', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); // fallback would be 6
+      mocks.websocket.serverSend.mockClear();
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 12);
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 12,
+      });
+    });
+
+    it('ignores the target segment for media segment requests (the filename wins)', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s', 99);
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 5,
+      });
+    });
   });
 
   describe('endSession', () => {

server/src/services/hls.service.ts

@@ -82,7 +82,14 @@ export class HlsService extends BaseService {
     return this.generateMediaPlaylist(asset);
   }
 
-  async getSegment(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, filename: string) {
+  async getSegment(
+    auth: AuthDto,
+    assetId: string,
+    sessionId: string,
+    variantIndex: number,
+    filename: string,
+    initSegment?: number,
+  ) {
     await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
 
     const session = await this.videoStreamRepository.getSession(sessionId);
@@ -99,7 +106,7 @@ export class HlsService extends BaseService {
     });
 
     const apiSession = this.trackSession(sessionId, variantIndex);
-    const segmentIndex = this.getSegmentIndex(apiSession, filename);
+    const segmentIndex = this.getSegmentIndex(apiSession, filename, initSegment);
     this.websocketRepository.serverSend('HlsHeartbeat', { sessionId, variantIndex, segmentIndex });
 
     if (await this.storageRepository.checkFileExists(path, constants.R_OK)) {
@@ -172,9 +179,13 @@ export class HlsService extends BaseService {
     return `${sessionId}:${variantIndex}:${segmentIndex}`;
   }
 
-  private getSegmentIndex(session: ApiSession, filename: string) {
+  private getSegmentIndex(session: ApiSession, filename: string, initSegment?: number) {
     if (filename.endsWith('.mp4')) {
-      return (session.lastRequestedSegment ?? -1) + 1;
+      // We need to know where to start transcoding, but the init.mp4 has no segment number in its name.
+      // We can infer this from the last requested segment, but this can be inaccurate given the client
+      // can load cached segments without reaching out to the server. `initSegment` acts as a hint to
+      // remove ambiguity when possible.
+      return initSegment ?? (session.lastRequestedSegment ?? -1) + 1;
     }
     const segmentIndex = Number.parseInt(HLS_SEGMENT_FILENAME_REGEX.exec(filename)![1]);
     session.lastRequestedSegment = segmentIndex;

server/src/services/transcoding.service.ts

@@ -30,6 +30,7 @@ type Session = {
   ownerId: string;
   paused: boolean;
   process: ChildProcess | null;
+  starting: boolean;
   startSegment: number | null;
   variantIndex: number | null;
 };
@@ -75,6 +76,7 @@ export class TranscodingService extends BaseService {
         ownerId,
         paused: false,
         process: null,
+        starting: false,
         startSegment: null,
         variantIndex: null,
       });
@@ -145,11 +147,19 @@ export class TranscodingService extends BaseService {
     } else if (session.process) {
       this.resumeTranscode(session);
       return;
+    } else if (session.starting) {
+      this.logger.debug(`Session ${sessionId} is already starting a transcode, skipping duplicate start request`);
+      return;
     }
 
-    const process = await this.startTranscode(session, variantIndex, segmentIndex);
-    if (process) {
-      session.process = process;
+    session.starting = true;
+    try {
+      const process = await this.startTranscode(session, variantIndex, segmentIndex);
+      if (process) {
+        session.process = process;
+      }
+    } finally {
+      session.starting = false;
     }
   }
 

web/package.json

@@ -40,14 +40,14 @@
     "@types/geojson": "^7946.0.16",
     "@zoom-image/core": "^0.42.0",
     "@zoom-image/svelte": "^0.3.0",
+    "custom-media-element": "^1.4.6",
     "dom-to-image": "^2.6.0",
     "fabric": "^7.0.0",
     "geo-coordinates-parser": "^1.7.4",
     "geojson": "^0.5.0",
     "handlebars": "^4.7.8",
     "happy-dom": "^20.0.0",
-    "hls-video-element": "^1.5.11",
-    "hls.js": "^1.6.16",
+    "hls.js": "1.7.0-beta.1.0.canary.11837",
     "intl-messageformat": "^11.0.0",
     "justified-layout": "^4.1.0",
     "lodash-es": "^4.17.21",

web/src/app.css

@@ -176,3 +176,9 @@
     @apply bg-subtle rounded-lg;
   }
 }
+
+immich-video > video {
+  width: 100%;
+  height: 100%;
+  object-fit: var(--media-object-fit, contain);
+}

web/src/lib/components/asset-viewer/VideoNativeViewer.svelte

@@ -5,9 +5,11 @@
   import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
   import { castManager } from '$lib/managers/cast-manager.svelte';
   import { featureFlagsManager } from '$lib/managers/feature-flags-manager.svelte';
-  import { mediaCapabilitiesManager } from '$lib/managers/media-capabilities-manager.svelte';
   import { autoPlayVideo, lang, loopVideo as loopVideoPreference } from '$lib/stores/preferences.store';
-  import { getAssetHlsSessionUrl, getAssetHlsUrl, getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
+  import { getAssetHlsUrl, getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
+  import '$lib/components/asset-viewer/immich-video-element';
+  import { videoSessionManager } from '$lib/managers/video-session-manager.svelte';
+  import VideoQualityMenu from '$lib/components/asset-viewer/VideoQualityMenu.svelte';
   import { AssetMediaSize, type AssetResponseDto } from '@immich/sdk';
   import { Icon, LoadingSpinner, shortcuts } from '@immich/ui';
   import {
@@ -23,9 +25,6 @@
     mdiVolumeMedium,
     mdiVolumeMute,
   } from '@mdi/js';
-  import 'hls-video-element';
-  import type HlsVideoElement from 'hls-video-element';
-  import Hls, { AbrController, Events, type FragLoadedData, type FragLoadingData, type HlsConfig } from 'hls.js';
   import 'media-chrome/media-control-bar';
   import 'media-chrome/media-controller';
   import 'media-chrome/media-fullscreen-button';
@@ -35,11 +34,10 @@
   import 'media-chrome/media-time-display';
   import 'media-chrome/media-volume-range';
   import 'media-chrome/menu/media-playback-rate-menu';
-  import 'media-chrome/menu/media-rendition-menu';
   import 'media-chrome/menu/media-settings-menu';
   import 'media-chrome/menu/media-settings-menu-button';
   import 'media-chrome/menu/media-settings-menu-item';
-  import { onDestroy, onMount } from 'svelte';
+  import { onMount } from 'svelte';
   import { useSwipe, type SwipeCustomEvent } from 'svelte-gestures';
   import { t } from 'svelte-i18n';
   import { fade } from 'svelte/transition';
@@ -73,7 +71,6 @@
     onClose = () => {},
   }: Props = $props();
 
-  let videoPlayer: HTMLVideoElement | undefined = $state();
   let isLoading = $state(true);
   let assetFileUrl = $derived.by(() => {
     if (featureFlagsManager.value.realtimeTranscoding) {
@@ -88,182 +85,29 @@
   });
   const aspectRatio = $derived(asset.width && asset.height ? `${asset.width} / ${asset.height}` : undefined);
   let showVideo = $state(false);
-  let hasFocused = $state(false);
-  let activeSession: { assetId: string; id: string } | undefined;
-  let rebuildCount = 0;
+  let focusedAssetId = $state<string>();
 
-  const MAX_REBUILDS = 1;
-  const SESSION_ID_REGEX = /\/video\/stream\/([0-9a-f-]{36})\//;
-
-  // hls.js can abandon fetching an in-flight fragment if it thinks it'll take too long, in which case
-  // it emergency switches to a different variant. This extends the delay even further due to
-  // cold starting another transcode, so let the fragment finish and have steady ABR decide the next level.
-  //
-  // It can also emergency switch between fragments: while a switch's first segment is still loading,
-  // it can run out of buffer and drop to a lower level for just one segment before continuing at the switched quality.
-  // This can cause multiple redundant transcoding restarts when it occurs.
-  // Hold the committed level until its first fragment lands, then resume normal ABR.
-  class NoAbandonAbrController extends AbrController {
-    private switchTarget = -1;
-
-    protected override onFragLoading(_event: Events.FRAG_LOADING, data: FragLoadingData) {
-      if (data.frag.sn === 'initSegment') {
-        this.switchTarget = data.frag.level;
-      }
-    }
-
-    protected override onFragLoaded(event: Events.FRAG_LOADED, data: FragLoadedData) {
-      if (data.frag.sn !== 'initSegment') {
-        this.switchTarget = -1;
-      }
-      super.onFragLoaded(event, data);
-    }
-
-    override get nextAutoLevel(): number {
-      const level = super.nextAutoLevel;
-      const target = this.hls.levels[this.switchTarget];
-      // Hold the committed level, but only while hls.js still considers it healthy.
-      if (target && level < this.switchTarget && target.loadError === 0 && target.fragmentError === 0) {
-        return this.switchTarget;
-      }
-      return level;
-    }
-
-    override set nextAutoLevel(level: number) {
-      super.nextAutoLevel = level;
-    }
-  }
-
-  const hlsConfig: Partial<HlsConfig> = {
-    abrController: NoAbandonAbrController,
-    highBufferWatchdogPeriod: 10,
-    detectStallWithCurrentTimeMs: 10_000,
-    maxBufferHole: 0.5,
-    maxBufferLength: 30,
-    maxMaxBufferLength: 60,
-    fragLoadPolicy: {
-      default: {
-        maxTimeToFirstByteMs: 30_000,
-        maxLoadTimeMs: 60_000,
-        timeoutRetry: { maxNumRetry: 5, retryDelayMs: 100, maxRetryDelayMs: 0 },
-        errorRetry: { maxNumRetry: 3, retryDelayMs: 1000, maxRetryDelayMs: 8000 },
-      },
-    },
-    useMediaCapabilities: false,
-  };
-
-  const releaseSession = () => {
-    const session = activeSession;
-    if (!session) {
-      return;
-    }
-    activeSession = undefined;
-    const url = getAssetHlsSessionUrl(session.assetId, session.id);
-    void fetch(url, { method: 'DELETE' }).catch(() => console.warn('Failed to release HLS session', session));
-  };
-
-  const isHlsElement = (el: HTMLVideoElement | undefined): el is HlsVideoElement => {
-    return el?.tagName === 'HLS-VIDEO';
-  };
-
-  const wireHlsListeners = (el: HlsVideoElement, assetId: string, resumeTime?: number) => {
-    const api = el.api;
-    if (!api) {
-      return;
-    }
-
-    // This is a hack to make the rendition menu use `api.currentLevel` instead of `api.nextLevel`.
-    // `api.nextLevel` makes the player request the next segment followed by the current segment.
-    // That backward request causes the server to restart transcoding for no reason.
-    Object.defineProperty(api, 'nextLevel', {
-      configurable: true,
-      get: () => api.currentLevel,
-      set: (level: number) => {
-        api.currentLevel = level;
-      },
-    });
-
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises
-    api.on(Hls.Events.MANIFEST_PARSED, async () => {
-      // Defer hls.js's first fragment load until we filter out suboptimal variants
-      api.stopLoad();
-      const id = api.levels[0]?.url[0]?.match(SESSION_ID_REGEX)?.[1];
-      if (id) {
-        activeSession = { assetId, id };
-      }
-
-      const keep = await mediaCapabilitiesManager.efficientLevels(api.levels);
-      for (let i = api.levels.length - 1; i >= 0; i--) {
-        if (!keep.has(i)) {
-          api.removeLevel(i);
-        }
-      }
-
-      api.startLoad(resumeTime);
-    });
-
-    api.on(Hls.Events.FRAG_LOADED, () => (rebuildCount = 0));
-
-    api.on(Hls.Events.ERROR, (_, data) => {
-      // 404 on a fragment can mean the server-side session has expired. Refetch
-      // master for a new session, but give up if it still 404s.
-      if (
-        !data.fatal ||
-        data.details !== Hls.ErrorDetails.FRAG_LOAD_ERROR ||
-        data.response?.code !== 404 ||
-        rebuildCount++ >= MAX_REBUILDS
-      ) {
-        console.error('HLS error', JSON.stringify(data));
-        return;
-      }
-      console.warn('Error loading segment, starting new session');
-      activeSession = undefined;
-      resumeTime = el.currentTime;
-      el.load();
-      // wireHlsListeners must run after el.api is repopulated.
-      queueMicrotask(() => wireHlsListeners(el, assetId, resumeTime));
-    });
-  };
+  const controller = $derived(videoSessionManager.get(assetId)); // <immich-video> self-acquires the controller for the asset
+  const videoPlayer = $derived(controller?.element);
 
   onMount(() => {
     showVideo = true;
   });
 
+  // A hover-warmed element is already past `canplay` and won't fire it again, so kick playback ourselves once we adopt it
   $effect(() => {
-    // reactive on `assetFileUrl` changes
-    if (videoPlayer && assetFileUrl) {
-      hasFocused = false;
-      rebuildCount = 0;
-      releaseSession();
-      if (isHlsElement(videoPlayer)) {
-        videoPlayer.config = hlsConfig;
-        videoPlayer.src = assetFileUrl;
-        const el = videoPlayer;
-        queueMicrotask(() => wireHlsListeners(el, assetId));
-      } else {
-        videoPlayer.load();
-      }
+    if (videoPlayer && videoPlayer.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {
+      void handleCanPlay(videoPlayer);
     }
-    return releaseSession;
   });
 
-  const onPagehide = (event: PageTransitionEvent) => {
-    if (!event.persisted) {
-      releaseSession();
+  const onPlaying = () => {
+    if (focusedAssetId !== assetId) {
+      videoPlayer?.focus();
+      focusedAssetId = assetId;
     }
   };
 
-  $effect(() => {
-    window.addEventListener('pagehide', onPagehide);
-    return () => window.removeEventListener('pagehide', onPagehide);
-  });
-
-  onDestroy(() => {
-    if (videoPlayer) {
-      videoPlayer.src = '';
-    }
-  });
-
   const handleCanPlay = async (video: HTMLVideoElement) => {
     try {
       if (!video.paused) {
@@ -352,53 +196,23 @@
         class="dark h-full max-w-full"
         style:aspect-ratio={aspectRatio}
         defaultduration={asset.duration! / 1000}
+        {...useSwipe(onSwipe)}
       >
-        {#if featureFlagsManager.value.realtimeTranscoding}
-          <hls-video
-            bind:this={videoPlayer}
-            slot="media"
-            loop={$loopVideoPreference && loopVideo}
-            autoplay={$autoPlayVideo}
-            disablePictureInPicture
-            playsinline
-            {...useSwipe(onSwipe)}
-            class="h-full object-contain"
-            oncanplay={(e: Event) => handleCanPlay(e.currentTarget as HTMLVideoElement)}
-            onended={onVideoEnded}
-            onseeking={onSeeking}
-            onplaying={(e: Event) => {
-              if (!hasFocused) {
-                (e.currentTarget as HTMLElement).focus();
-                hasFocused = true;
-              }
-            }}
-            onclose={onClose}
-            poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })}
-          ></hls-video>
-        {:else}
-          <video
-            bind:this={videoPlayer}
-            slot="media"
-            src={assetFileUrl}
-            loop={$loopVideoPreference && loopVideo}
-            autoplay={$autoPlayVideo}
-            disablePictureInPicture
-            playsinline
-            {...useSwipe(onSwipe)}
-            class="h-full object-contain"
-            oncanplay={(e) => handleCanPlay(e.currentTarget)}
-            onended={onVideoEnded}
-            onseeking={onSeeking}
-            onplaying={(e) => {
-              if (!hasFocused) {
-                e.currentTarget.focus();
-                hasFocused = true;
-              }
-            }}
-            onclose={onClose}
-            poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })}
-          ></video>
-        {/if}
+        <immich-video
+          slot="media"
+          asset-id={assetId}
+          cache-key={cacheKey ?? ''}
+          play-original={playOriginalVideo}
+          class="h-full"
+          loop={$loopVideoPreference && loopVideo}
+          autoplay={$autoPlayVideo}
+          poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })}
+          oncanplay={(event: Event) => handleCanPlay(event.currentTarget as HTMLVideoElement)}
+          onended={onVideoEnded}
+          onseeking={onSeeking}
+          onplaying={onPlaying}
+          onclose={onClose}
+        ></immich-video>
 
         {#if extendedControls}
           <media-settings-menu hidden anchor="auto" class="min-w-3xs rounded-xl border border-light-300 shadow-sm">
@@ -411,14 +225,11 @@
                 <span slot="title">{$t('media_chrome.playback_rate')}</span>
               </media-playback-rate-menu>
             </media-settings-menu-item>
-            {#if featureFlagsManager.value.realtimeTranscoding}
+            {#if featureFlagsManager.value.realtimeTranscoding && controller}
               <media-settings-menu-item class="mx-1 rounded-lg p-1 ps-2">
                 {$t('video_quality')}
                 <Icon slot="suffix" icon={mdiChevronRight} class="m-2" />
-                <media-rendition-menu slot="submenu" hidden>
-                  <Icon slot="back-icon" icon={mdiChevronLeft} class="m-2" />
-                  <span slot="title">{$t('video_quality')}</span>
-                </media-rendition-menu>
+                <VideoQualityMenu video={controller} slot="submenu" />
               </media-settings-menu-item>
             {/if}
           </media-settings-menu>

web/src/lib/components/asset-viewer/VideoQualityMenu.svelte

@@ -0,0 +1,57 @@
+<script lang="ts">
+  import type { VideoController } from '$lib/utils/video/controller.svelte';
+  import { Icon } from '@immich/ui';
+  import { mdiChevronLeft } from '@mdi/js';
+  import 'media-chrome/menu/media-chrome-menu';
+  import 'media-chrome/menu/media-chrome-menu-item';
+  import { t } from 'svelte-i18n';
+  import type { HTMLAttributes } from 'svelte/elements';
+
+  interface Props extends HTMLAttributes<HTMLElement> {
+    video: VideoController;
+  }
+
+  let { video, ...rest }: Props = $props();
+
+  const options = $derived(
+    video.levels
+      .map((level, idx) => ({ idx, label: Math.min(level.width, level.height) }))
+      .sort((a, b) => b.label - a.label),
+  );
+
+  const autoLevel = $derived(video.selectedLevel === -1 ? options.find(({ idx }) => idx === video.level) : undefined);
+  const autoLabel = $derived(autoLevel ? `${$t('media_chrome.auto')} (${autoLevel.label}p)` : $t('media_chrome.auto'));
+
+  let menu = $state<HTMLElement>();
+  $effect(() => {
+    menu?.dispatchEvent(new CustomEvent('addmenuitem', { detail: autoLabel }));
+  });
+
+  const onChange = (event: Event) => {
+    video.level = Number((event.currentTarget as HTMLElement & { value: string }).value);
+  };
+</script>
+
+<media-chrome-menu bind:this={menu} {...rest} hidden onchange={onChange}>
+  <Icon slot="back-icon" icon={mdiChevronLeft} class="m-2" />
+  <span slot="title">{$t('video_quality')}</span>
+  <media-chrome-menu-item part="menu-item radio" type="radio" value="-1" checked={video.selectedLevel === -1}>
+    <span>{autoLabel}</span>
+  </media-chrome-menu-item>
+  {#each options as option (option.idx)}
+    <media-chrome-menu-item
+      part="menu-item radio"
+      type="radio"
+      value={`${option.idx}`}
+      checked={video.selectedLevel === option.idx}
+    >
+      <span>{option.label}p</span>
+    </media-chrome-menu-item>
+  {/each}
+</media-chrome-menu>
+
+<style>
+  media-chrome-menu-item {
+    padding: 0.4em 0.8em 0.4em 1em;
+  }
+</style>

web/src/lib/components/asset-viewer/immich-video-element.ts

@@ -0,0 +1,84 @@
+import { CustomVideoElement } from 'custom-media-element';
+import { videoSessionManager } from '$lib/managers/video-session-manager.svelte';
+import type { VideoController } from '$lib/utils/video/controller.svelte';
+
+/**
+ * Video backed by either HLS or a progressive stream based on feature flags and user preferences. Can be managed with
+ * `videoSessionManager.get(assetId)`, this manager being what allows it to reparent the underlying video element.
+ */
+class ImmichVideoElement extends CustomVideoElement {
+  static override get observedAttributes() {
+    return [...super.observedAttributes, 'asset-id', 'play-original'];
+  }
+
+  #controller: VideoController | undefined;
+  #mountedAssetId: string | undefined;
+  #remountScheduled = false;
+
+  override connectedCallback() {
+    super.connectedCallback();
+    this.#mount();
+  }
+
+  override disconnectedCallback() {
+    super.disconnectedCallback();
+    this.#unmount();
+  }
+
+  override attributeChangedCallback(name: string, oldValue: string | null, newValue: string | null) {
+    super.attributeChangedCallback(name, oldValue, newValue);
+    if (!this.isConnected || !this.#controller || oldValue === newValue) {
+      return;
+    }
+    if (name === 'play-original') {
+      this.#controller.playOriginal = newValue === 'true';
+    } else if (name === 'asset-id' && !this.#remountScheduled) {
+      this.#remountScheduled = true;
+      queueMicrotask(() => {
+        this.#remountScheduled = false;
+        if (this.isConnected) {
+          this.#unmount();
+          this.#mount();
+        }
+      });
+    }
+  }
+
+  #mount() {
+    const assetId = this.getAttribute('asset-id');
+    if (!assetId) {
+      return;
+    }
+    const controller = videoSessionManager.acquire({
+      assetId,
+      cacheKey: this.getAttribute('cache-key') || null,
+      playOriginal: this.getAttribute('play-original') === 'true',
+    });
+    this.#controller = controller;
+    this.#mountedAssetId = assetId;
+
+    const video = controller.element;
+    video.slot = 'media';
+    video.loop = this.loop;
+    video.autoplay = this.autoplay;
+    video.muted = this.muted || this.hasAttribute('muted');
+    video.poster = this.getAttribute('poster') ?? '';
+    controller.mount(this);
+  }
+
+  #unmount() {
+    if (!this.#mountedAssetId) {
+      return;
+    }
+    this.#controller?.unmount(this);
+    videoSessionManager.release(this.#mountedAssetId);
+    this.#controller = undefined;
+    this.#mountedAssetId = undefined;
+  }
+}
+
+if (globalThis.customElements && !customElements.get('immich-video')) {
+  customElements.define('immich-video', ImmichVideoElement);
+}
+
+export default ImmichVideoElement;

web/src/lib/components/assets/thumbnail/Thumbnail.svelte

@@ -4,7 +4,7 @@
   import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
   import { mediaQueryManager } from '$lib/stores/media-query-manager.svelte';
   import { locale, playVideoThumbnailOnHover } from '$lib/stores/preferences.store';
-  import { getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
+  import { getAssetMediaUrl } from '$lib/utils';
   import { moveFocus } from '$lib/utils/focus-util';
   import { currentUrlReplaceAssetId } from '$lib/utils/navigation';
   import { getAltText } from '$lib/utils/thumbnail-util';
@@ -264,7 +264,8 @@
         <div class="pointer-events-none absolute size-full group-focus-visible:rounded-lg">
           <VideoThumbnail
             class="group-focus-visible:rounded-lg"
-            url={getAssetPlaybackUrl({ id: asset.id, cacheKey: asset.thumbhash })}
+            assetId={asset.id}
+            cacheKey={asset.thumbhash}
             enablePlayback={mouseOver && $playVideoThumbnailOnHover}
             curve={selected}
             durationInSeconds={asset.duration ? asset.duration / 1000 : 0}
@@ -275,7 +276,8 @@
         <div class="pointer-events-none absolute size-full group-focus-visible:rounded-lg">
           <VideoThumbnail
             class="group-focus-visible:rounded-lg"
-            url={getAssetPlaybackUrl({ id: asset.livePhotoVideoId, cacheKey: asset.thumbhash })}
+            assetId={asset.livePhotoVideoId}
+            cacheKey={asset.thumbhash}
             enablePlayback={mouseOver && $playVideoThumbnailOnHover}
             pauseIcon={mdiMotionPauseOutline}
             playIcon={mdiMotionPlayOutline}

web/src/lib/components/assets/thumbnail/VideoThumbnail.svelte

@@ -1,12 +1,16 @@
 <script lang="ts">
   import { cleanClass } from '$lib';
+  import '$lib/components/asset-viewer/immich-video-element';
+  import { featureFlagsManager } from '$lib/managers/feature-flags-manager.svelte';
+  import { videoSessionManager } from '$lib/managers/video-session-manager.svelte';
   import { Icon, LoadingSpinner } from '@immich/ui';
   import { mdiAlertCircleOutline, mdiPauseCircleOutline, mdiPlayCircleOutline } from '@mdi/js';
   import { Duration } from 'luxon';
   import type { ClassValue } from 'svelte/elements';
 
   interface Props {
-    url: string;
+    assetId: string;
+    cacheKey: string | null;
     durationInSeconds?: number;
     enablePlayback?: boolean;
     playbackOnIconHover?: boolean;
@@ -18,7 +22,8 @@
   }
 
   let {
-    url,
+    assetId,
+    cacheKey,
     durationInSeconds = 0,
     enablePlayback = $bindable(false),
     playbackOnIconHover = false,
@@ -29,26 +34,27 @@
     class: className,
   }: Props = $props();
 
-  let remainingSeconds = $state(durationInSeconds);
-  let loading = $state(true);
-  let error = $state(false);
-  let player: HTMLVideoElement | undefined = $state();
+  const useHls = $derived(featureFlagsManager.value.realtimeTranscoding);
+
+  let active = $state(false);
+  const controller = $derived(videoSessionManager.get(assetId));
+  const remainingSeconds = $derived(controller?.remainingSeconds || durationInSeconds);
 
   $effect(() => {
     if (!enablePlayback) {
-      remainingSeconds = durationInSeconds;
+      active = false;
       return;
     }
-    if (!player) {
+    if (!useHls) {
+      active = true;
       return;
     }
-    const video = player;
-    return () => {
-      video.pause();
-      video.removeAttribute('src');
-      video.load();
-    };
+    // Cold-starting a transcode for every thumbnail the pointer brushes over would hammer the server,
+    // so wait for the hover to settle before opening an HLS session.
+    const timer = setTimeout(() => (active = true), 200);
+    return () => clearTimeout(timer);
   });
+
   const onMouseEnter = () => {
     if (playbackOnIconHover) {
       enablePlayback = true;
@@ -62,35 +68,15 @@
   };
 </script>
 
-{#if enablePlayback}
-  <video
-    bind:this={player}
-    class={cleanClass('h-full w-full object-cover', className)}
-    class:rounded-xl={curve}
+{#if active}
+  <immich-video
+    asset-id={assetId}
+    cache-key={cacheKey ?? ''}
     muted
-    autoplay
     loop
-    src={url}
-    onplay={() => {
-      loading = false;
-      error = false;
-    }}
-    onerror={() => {
-      if (!player?.src) {
-        // Do not show error when the URL is empty.
-        return;
-      }
-      error = true;
-      loading = false;
-    }}
-    ontimeupdate={({ currentTarget }) => {
-      const remaining = currentTarget.duration - currentTarget.currentTime;
-      remainingSeconds = Math.min(
-        Math.ceil(Number.isNaN(remaining) ? Number.POSITIVE_INFINITY : remaining),
-        durationInSeconds,
-      );
-    }}
-  ></video>
+    autoplay
+    class={cleanClass('h-full w-full [--media-object-fit:cover]', className, curve && 'rounded-xl overflow-hidden')}
+  ></immich-video>
 {/if}
 
 <div
@@ -114,10 +100,10 @@
     onmouseenter={onMouseEnter}
     onmouseleave={onMouseLeave}
   >
-    {#if enablePlayback}
-      {#if loading}
+    {#if active}
+      {#if !controller || controller.loading}
         <LoadingSpinner size="large" />
-      {:else if error}
+      {:else if controller.error}
         <Icon icon={mdiAlertCircleOutline} size="24" class="text-red-600" />
       {:else}
         <Icon icon={pauseIcon} size="24" />

web/src/lib/managers/video-session-manager.svelte.ts

@@ -0,0 +1,47 @@
+import { SvelteMap } from 'svelte/reactivity';
+import { VideoController, type VideoControllerOptions } from '$lib/utils/video/controller.svelte';
+
+interface Session {
+  controller: VideoController;
+  refs: number;
+  timer?: NodeJS.Timeout;
+}
+
+/**
+ * Registry of controllers keyed by asset, ref-counted with a grace period. `<immich-video>` acquires and
+ * releases as it connects and disconnects, with controllers kept briefly before being disposed. This enables
+ * reuse of bandwidth estimation, downloaded segments, HLS session, etc. for seamless handoff.
+ */
+class VideoSessionManager {
+  #sessions = new SvelteMap<string, Session>();
+
+  acquire(options: VideoControllerOptions): VideoController {
+    const existing = this.#sessions.get(options.assetId);
+    if (existing) {
+      clearTimeout(existing.timer);
+      existing.timer = undefined;
+      existing.refs++;
+      return existing.controller;
+    }
+    const controller = new VideoController(options);
+    this.#sessions.set(options.assetId, { controller, refs: 1, timer: undefined });
+    return controller;
+  }
+
+  release(assetId: string) {
+    const session = this.#sessions.get(assetId);
+    if (!session || --session.refs > 0) {
+      return;
+    }
+    session.timer = setTimeout(() => {
+      session.controller.release();
+      this.#sessions.delete(assetId);
+    }, 1_000);
+  }
+
+  get(assetId: string): VideoController | undefined {
+    return this.#sessions.get(assetId)?.controller;
+  }
+}
+
+export const videoSessionManager = new VideoSessionManager();

web/src/lib/utils/video/controller.svelte.ts

@@ -0,0 +1,219 @@
+import { AssetMediaSize } from '@immich/sdk';
+import Hls, { type ErrorData, type Level } from 'hls.js';
+import { featureFlagsManager } from '$lib/managers/feature-flags-manager.svelte';
+import { getAssetHlsUrl, getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
+import { createHls, filterEfficientLevels, getHlsSessionId, releaseHlsSession } from '$lib/utils/video/hls';
+
+export interface VideoControllerOptions {
+  assetId: string;
+  cacheKey: string | null;
+  playOriginal: boolean;
+}
+
+const HLS_MIME = 'application/x-mpegURL';
+const MAX_REBUILDS = 1;
+
+/**
+ * Owns a single, long-lived `<video>` for an asset and all of its playback wiring.
+ * Because the controller owns the element, hosts can {@link mount} it and hand it off by re-parenting, enabling
+ * bandwidth estimation, buffer, HLS session, playback position, etc. to survive.
+ */
+export class VideoController {
+  readonly element: HTMLVideoElement;
+
+  loading = $state(true);
+  error = $state(false);
+  currentTime = $state(0);
+  duration = $state(0);
+  levels = $state<Level[]>([]);
+  selectedLevel = $state(-1);
+
+  private assetId: string;
+  private cacheKey: string | null;
+  private api: Hls | undefined;
+  private sourceTeardown: (() => void) | undefined;
+  private started = false;
+  private wasPlaying = false;
+  private rebuilds = 0;
+
+  #level = $state(-1);
+  #playOriginal: boolean;
+
+  constructor({ assetId, cacheKey, playOriginal }: VideoControllerOptions) {
+    this.assetId = assetId;
+    this.cacheKey = cacheKey;
+    this.#playOriginal = playOriginal;
+
+    const element = document.createElement('video');
+    element.playsInline = true;
+    element.disablePictureInPicture = true;
+    element.addEventListener('play', () => {
+      this.loading = false;
+      this.error = false;
+    });
+    element.addEventListener('error', () => {
+      this.error = true;
+      this.loading = false;
+    });
+    element.addEventListener('timeupdate', () => {
+      this.currentTime = element.currentTime;
+      this.duration = element.duration;
+    });
+    this.element = element;
+  }
+
+  mount(container: HTMLElement) {
+    container.append(this.element);
+    if (!this.started) {
+      this.started = true;
+      const useHls = featureFlagsManager.value.realtimeTranscoding && !this.#playOriginal;
+      this.sourceTeardown = useHls ? this.attachHls() : this.attachProgressive();
+    } else if (this.wasPlaying) {
+      void this.element.play().catch(() => {});
+    }
+  }
+
+  unmount(container: HTMLElement) {
+    if (this.element.parentElement === container) {
+      this.wasPlaying = !this.element.paused;
+      this.element.remove();
+    }
+  }
+
+  release() {
+    this.sourceTeardown?.();
+    this.sourceTeardown = undefined;
+    this.element.remove();
+  }
+
+  get remainingSeconds() {
+    return Math.max(0, this.duration - this.currentTime);
+  }
+
+  set playOriginal(playOriginal: boolean) {
+    if (this.#playOriginal === playOriginal) {
+      return;
+    }
+    this.#playOriginal = playOriginal;
+    if (!this.started) {
+      return;
+    }
+    this.sourceTeardown?.();
+    const useHls = featureFlagsManager.value.realtimeTranscoding && !playOriginal;
+    this.sourceTeardown = useHls ? this.attachHls() : this.attachProgressive();
+  }
+
+  get level() {
+    return this.#level;
+  }
+
+  set level(index: number) {
+    if (!this.api) {
+      return;
+    }
+    // -1 re-enables ABR without flushing
+    if (index === -1) {
+      this.api.loadLevel = -1;
+    } else {
+      this.api.currentLevel = index;
+    }
+    this.selectedLevel = index;
+  }
+
+  private attachProgressive() {
+    this.element.src = this.#playOriginal
+      ? getAssetMediaUrl({ id: this.assetId, size: AssetMediaSize.Original, cacheKey: this.cacheKey })
+      : getAssetPlaybackUrl({ id: this.assetId, cacheKey: this.cacheKey });
+    return () => this.detachSource();
+  }
+
+  private detachSource() {
+    this.element.pause();
+    this.element.removeAttribute('src');
+    this.element.load();
+  }
+
+  private attachHls(startPosition = -1): () => void {
+    const video = this.element;
+    // Old iOS versions don't support Media Source Extensions
+    if (!Hls.isSupported()) {
+      return this.attachNativeHls();
+    }
+
+    const hls = createHls({ autoStartLoad: false });
+    this.api = hls;
+    let sessionId: string | undefined;
+
+    // eslint-disable-next-line @typescript-eslint/no-misused-promises
+    hls.on(Hls.Events.MANIFEST_PARSED, async () => {
+      sessionId = getHlsSessionId(hls);
+      await filterEfficientLevels(hls);
+      this.levels = hls.levels;
+      hls.attachMedia(video); // Need to attach after filtering for the auto size cap to work
+      hls.startLoad(startPosition);
+      // autoStartLoad defers the first fragment, so the `autoplay` attribute may have already fired and done nothing
+      if (video.autoplay && video.paused) {
+        void video.play().catch(() => {});
+      }
+    });
+    hls.on(Hls.Events.LEVELS_UPDATED, (_, data) => (this.levels = data.levels));
+    hls.on(Hls.Events.LEVEL_SWITCHED, (_, data) => (this.#level = data.level));
+    hls.on(Hls.Events.FRAG_LOADED, () => (this.rebuilds = 0));
+    hls.on(Hls.Events.ERROR, (_, data) => this.onHlsError(data));
+
+    const onPageHide = (event: PageTransitionEvent) => {
+      if (!event.persisted && sessionId) {
+        releaseHlsSession(this.assetId, sessionId);
+      }
+    };
+    window.addEventListener('pagehide', onPageHide);
+
+    hls.loadSource(getAssetHlsUrl(this.assetId));
+
+    return () => {
+      window.removeEventListener('pagehide', onPageHide);
+      if (sessionId) {
+        releaseHlsSession(this.assetId, sessionId);
+      }
+      hls.destroy();
+      this.api = undefined;
+      this.levels = [];
+      this.#level = -1;
+      this.selectedLevel = -1;
+    };
+  }
+
+  private attachNativeHls() {
+    if (this.element.canPlayType(HLS_MIME)) {
+      this.element.src = getAssetHlsUrl(this.assetId);
+    } else {
+      this.error = true;
+      this.loading = false;
+    }
+    return () => this.detachSource();
+  }
+
+  private onHlsError(data: ErrorData) {
+    // A fragment 404 usually means the server session expired (e.g. after a long pause). Rebuild it
+    // once, resuming where we left off, before giving up.
+    if (
+      data.fatal &&
+      data.details === Hls.ErrorDetails.FRAG_LOAD_ERROR &&
+      data.response?.code === 404 &&
+      this.rebuilds < MAX_REBUILDS
+    ) {
+      this.rebuilds++;
+      this.loading = true;
+      this.error = false;
+      const resume = this.element.currentTime;
+      this.sourceTeardown?.();
+      this.sourceTeardown = this.attachHls(resume);
+      return;
+    }
+    if (data.fatal) {
+      console.error('Fatal HLS error', data.details, data.response?.code);
+      this.error = true;
+      this.loading = false;
+    }
+  }
+}

web/src/lib/utils/video/hls.ts

@@ -0,0 +1,157 @@
+import Hls, {
+  AbrController,
+  Events,
+  FetchLoader,
+  type FragLoadedData,
+  type FragLoadingData,
+  type HlsConfig,
+} from 'hls.js';
+import { debounce } from 'lodash-es';
+import { mediaCapabilitiesManager } from '$lib/managers/media-capabilities-manager.svelte';
+import { getAssetHlsSessionUrl } from '$lib/utils';
+
+const HLS_TARGET_SEGMENT_HEADER = 'x-immich-hls-msn';
+const RESIZE_FLUSH_DEBOUNCE_MS = 150;
+const SESSION_ID_REGEX = /\/video\/stream\/([0-9a-f-]{36})\//;
+
+// hls.js can abandon fetching an in-flight fragment if it thinks it'll take too long, in which case
+// it emergency switches to a different variant. This extends the delay even further due to
+// cold starting another transcode, so let the fragment finish and have steady ABR decide the next level.
+//
+// It can also emergency switch between fragments: while a switch's first segment is still loading,
+// it can run out of buffer and drop to a lower level for just one segment before continuing at the switched quality.
+// This can cause multiple redundant transcoding restarts when it occurs.
+// Hold the committed level until its first fragment lands, then resume normal ABR.
+export class NoAbandonAbrController extends AbrController {
+  private switchTarget = -1;
+
+  protected override onFragLoading(_event: Events.FRAG_LOADING, data: FragLoadingData) {
+    if (data.frag.sn === 'initSegment') {
+      this.switchTarget = data.frag.level;
+    }
+  }
+
+  protected override onFragLoaded(event: Events.FRAG_LOADED, data: FragLoadedData) {
+    if (data.frag.sn !== 'initSegment') {
+      this.switchTarget = -1;
+    }
+    super.onFragLoaded(event, data);
+  }
+
+  override get nextAutoLevel(): number {
+    const level = super.nextAutoLevel;
+    const target = this.hls.levels[this.switchTarget];
+    // Hold the committed level, but only while hls.js still considers it healthy.
+    if (target && level < this.switchTarget && target.loadError === 0 && target.fragmentError === 0) {
+      return this.switchTarget;
+    }
+    return level;
+  }
+
+  override set nextAutoLevel(level: number) {
+    super.nextAutoLevel = level;
+  }
+}
+
+// hls.js flushes the forward buffer on a level switch so the new variant surfaces quickly, but requests can happen
+// out of order and cause unnecessary transcodes since it leaves the loader running during the flush.
+// This version stops the loader until the buffer is flushed so segment requests stay monotonic.
+class FlushAheadStreamController extends Hls.DefaultConfig.streamController {
+  #flushPending = false;
+  #flushAhead = debounce(() => this.#switchAhead(), RESIZE_FLUSH_DEBOUNCE_MS);
+
+  override nextLevelSwitch() {
+    this.#flushAhead();
+  }
+
+  #switchAhead() {
+    const { media, hls, levels, playlistType } = this;
+    if (!media?.readyState || !levels || this.#flushPending) {
+      return;
+    }
+    const bufferInfo = this.getFwdBufferInfo(this.getBufferOutput(), playlistType);
+    const nextLevel = levels[hls.nextLoadLevel];
+    if (!bufferInfo || !nextLevel) {
+      return;
+    }
+    const { fetchdelay, okToFlushForwardBuffer } = this.calculateOptimalSwitchPoint(nextLevel, bufferInfo);
+    const flushFrom = this.playhead + fetchdelay;
+    if (!okToFlushForwardBuffer || bufferInfo.end <= flushFrom) {
+      return;
+    }
+    this.#flushPending = true;
+    hls.stopLoad();
+    hls.once(Events.BUFFER_FLUSHED, () => {
+      this.#flushPending = false;
+      hls.startLoad();
+    });
+    hls.trigger(Events.BUFFER_FLUSHING, {
+      startOffset: flushFrom,
+      endOffset: Number.POSITIVE_INFINITY,
+      type: null,
+    });
+  }
+}
+
+export const createHls = (overrides?: Partial<HlsConfig>): Hls => {
+  const hls = new Hls({
+    abrController: NoAbandonAbrController,
+    loader: FetchLoader,
+    capLevelToPlayerSize: true,
+    streamController: FlushAheadStreamController,
+    testBandwidth: false,
+    highBufferWatchdogPeriod: 10,
+    detectStallWithCurrentTimeMs: 10_000,
+    maxBufferHole: 0.5,
+    maxBufferLength: 30,
+    maxMaxBufferLength: 60,
+    fragLoadPolicy: {
+      default: {
+        maxTimeToFirstByteMs: 30_000,
+        maxLoadTimeMs: 60_000,
+        timeoutRetry: { maxNumRetry: 5, retryDelayMs: 100, maxRetryDelayMs: 0 },
+        errorRetry: { maxNumRetry: 3, retryDelayMs: 1000, maxRetryDelayMs: 8000 },
+      },
+    },
+    useMediaCapabilities: false,
+    ...overrides,
+  });
+
+  // init.mp4 carries no segment number, but the server needs to know which segment an init.mp4
+  // is for so it can start the transcode there. It sometimes can't infer this since segments can
+  // be loaded from browser cache and the server might not know where the client really is as a result.
+  // Let the client hint the target segment it's switching to with a custom header.
+  hls.config.fetchSetup = (context, initParams) => {
+    const frag = (context as { frag?: { sn: number | 'initSegment' } }).frag;
+    if (frag?.sn === 'initSegment') {
+      const sn = hls.inFlightFragments.main.frag?.sn;
+      if (typeof sn === 'number') {
+        (initParams.headers as Headers).set(HLS_TARGET_SEGMENT_HEADER, String(sn));
+      }
+    }
+    return new Request(context.url, initParams);
+  };
+
+  return hls;
+};
+
+export const getHlsSessionId = (api: Hls): string | undefined => {
+  return api.levels[0]?.url[0]?.match(SESSION_ID_REGEX)?.[1];
+};
+
+export const releaseHlsSession = (assetId: string, sessionId: string) => {
+  const url = getAssetHlsSessionUrl(assetId, sessionId);
+  void fetch(url, { method: 'DELETE' }).catch(() =>
+    console.warn('Failed to release HLS session', { assetId, sessionId }),
+  );
+};
+
+/** Drop every variant the browser can't hardware-decode efficiently, keeping one per resolution. */
+export const filterEfficientLevels = async (api: Hls) => {
+  const keep = await mediaCapabilitiesManager.efficientLevels(api.levels);
+  for (let i = api.levels.length - 1; i >= 0; i--) {
+    if (!keep.has(i)) {
+      api.removeLevel(i);
+    }
+  }
+};

web/src/routes/(user)/memory/[[photos=photos]]/[[assetId=id]]/MemoryVideoViewer.svelte

@@ -1,8 +1,10 @@
 <script lang="ts">
+  import '$lib/components/asset-viewer/immich-video-element';
   import { assetViewerFadeDuration } from '$lib/constants';
   import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
   import { autoPlayVideo } from '$lib/stores/preferences.store';
-  import { getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
+  import { getAssetMediaUrl } from '$lib/utils';
+  import { videoSessionManager } from '$lib/managers/video-session-manager.svelte';
   import { AssetMediaSize } from '@immich/sdk';
   import 'media-chrome/media-controller';
   import { onMount } from 'svelte';
@@ -15,29 +17,34 @@
 
   let { asset, videoPlayer = $bindable() }: Props = $props();
 
-  let showVideo: boolean = $state(false);
-
+  let showVideo = $state(false);
   onMount(() => {
     // Show video after mount to ensure fading in.
     showVideo = true;
   });
+
+  const controller = $derived(videoSessionManager.get(asset.id)); // <immich-video> self-acquires the controller for the asset
+  $effect(() => {
+    videoPlayer = controller?.element;
+    return () => {
+      videoPlayer = undefined;
+    };
+  });
 </script>
 
 {#if showVideo}
-  <div class="bg-pink-9000 size-full" transition:fade={{ duration: assetViewerFadeDuration }}>
+  <div class="size-full" transition:fade={{ duration: assetViewerFadeDuration }}>
+    <!-- svelte-ignore a11y_media_has_caption -->
     <media-controller id="memory-video" nohotkeys class="size-full rounded-2xl object-contain transition-all">
-      <!-- svelte-ignore a11y_media_has_caption -->
-      <video
-        bind:this={videoPlayer}
+      <immich-video
         slot="media"
-        autoplay={$autoPlayVideo}
-        playsinline
-        disablepictureinpicture
+        asset-id={asset.id}
+        cache-key={asset.thumbhash ?? ''}
         class="size-full"
-        src={getAssetPlaybackUrl({ id: asset.id })}
-        poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview })}
         draggable="false"
-      ></video>
+        autoplay={$autoPlayVideo}
+        poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview })}
+      ></immich-video>
     </media-controller>
   </div>
 {/if}