diff --git a/i18n/en.json b/i18n/en.json index 934227f3b9..df284906b1 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -431,6 +431,10 @@ "transcoding_realtime_description": "Allows transcoding to be performed in real-time as the video is being streamed. Enables quality switching, but may cause higher playback latency and stuttering depending on server capabilities.", "transcoding_realtime_enabled": "Enable real-time transcoding", "transcoding_realtime_enabled_description": "If disabled, the server will refuse to start new real-time transcoding sessions.", + "transcoding_realtime_resolutions": "Resolutions", + "transcoding_realtime_resolutions_description": "The resolutions offered for real-time transcoding. A variant is only offered when its resolution is no larger than the source. Higher resolutions may cause playback issues if the server cannot transcode them quickly enough.", + "transcoding_realtime_video_codecs": "Video codecs", + "transcoding_realtime_video_codecs_description": "The video codecs offered for real-time transcoding. Clients will choose the best option they support during playback. AV1 is more efficient than HEVC, which is more efficient than H.264. When using hardware acceleration, only select the codecs the accelerator can encode. When using software transcoding, note that H.264 is faster than AV1, which is faster than HEVC.", "transcoding_reference_frames": "Reference frames", "transcoding_reference_frames_description": "The number of frames to reference when compressing a given frame. Higher values improve compression efficiency, but slow down encoding. 0 sets this value automatically.", "transcoding_required_description": "Only videos not in an accepted format", diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index ece44d310a..5a082cb1e0 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -452,6 +452,7 @@ Class | Method | HTTP request | Description - [FacialRecognitionConfig](doc//FacialRecognitionConfig.md) - [FoldersResponse](doc//FoldersResponse.md) - [FoldersUpdate](doc//FoldersUpdate.md) + - [HlsVideoResolution](doc//HlsVideoResolution.md) - [ImageFormat](doc//ImageFormat.md) - [IntegrityReport](doc//IntegrityReport.md) - [IntegrityReportResponseDto](doc//IntegrityReportResponseDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 4a20fe79a6..47425b56fe 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -173,6 +173,7 @@ part 'model/face_dto.dart'; part 'model/facial_recognition_config.dart'; part 'model/folders_response.dart'; part 'model/folders_update.dart'; +part 'model/hls_video_resolution.dart'; part 'model/image_format.dart'; part 'model/integrity_report.dart'; part 'model/integrity_report_response_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 137b0a47c5..873b05b8d2 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -391,6 +391,8 @@ class ApiClient { return FoldersResponse.fromJson(value); case 'FoldersUpdate': return FoldersUpdate.fromJson(value); + case 'HlsVideoResolution': + return HlsVideoResolutionTypeTransformer().decode(value); case 'ImageFormat': return ImageFormatTypeTransformer().decode(value); case 'IntegrityReport': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index a985dece82..c8efcb5466 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -106,6 +106,9 @@ String parameterToString(dynamic value) { if (value is Colorspace) { return ColorspaceTypeTransformer().encode(value).toString(); } + if (value is HlsVideoResolution) { + return HlsVideoResolutionTypeTransformer().encode(value).toString(); + } if (value is ImageFormat) { return ImageFormatTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/hls_video_resolution.dart b/mobile/openapi/lib/model/hls_video_resolution.dart new file mode 100644 index 0000000000..d28fb9055c --- /dev/null +++ b/mobile/openapi/lib/model/hls_video_resolution.dart @@ -0,0 +1,94 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +/// HLS video resolution +class HlsVideoResolution { + /// Instantiate a new enum with the provided [value]. + const HlsVideoResolution._(this.value); + + /// The underlying value of this enum member. + final int value; + + @override + String toString() => value.toString(); + + int toJson() => value; + + static const number480 = HlsVideoResolution._(480); + static const number720 = HlsVideoResolution._(720); + static const number1080 = HlsVideoResolution._(1080); + static const number1440 = HlsVideoResolution._(1440); + static const number2160 = HlsVideoResolution._(2160); + + /// List of all possible values in this [enum][HlsVideoResolution]. + static const values = [ + number480, + number720, + number1080, + number1440, + number2160, + ]; + + static HlsVideoResolution? fromJson(dynamic value) => HlsVideoResolutionTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = HlsVideoResolution.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [HlsVideoResolution] to int, +/// and [decode] dynamic data back to [HlsVideoResolution]. +class HlsVideoResolutionTypeTransformer { + factory HlsVideoResolutionTypeTransformer() => _instance ??= const HlsVideoResolutionTypeTransformer._(); + + const HlsVideoResolutionTypeTransformer._(); + + int encode(HlsVideoResolution data) => data.value; + + /// Decodes a [dynamic value][data] to a HlsVideoResolution. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + HlsVideoResolution? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case 480: return HlsVideoResolution.number480; + case 720: return HlsVideoResolution.number720; + case 1080: return HlsVideoResolution.number1080; + case 1440: return HlsVideoResolution.number1440; + case 2160: return HlsVideoResolution.number2160; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [HlsVideoResolutionTypeTransformer] instance. + static HlsVideoResolutionTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_realtime_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_realtime_dto.dart index 1a8669912e..e88cd0d5c1 100644 --- a/mobile/openapi/lib/model/system_config_f_fmpeg_realtime_dto.dart +++ b/mobile/openapi/lib/model/system_config_f_fmpeg_realtime_dto.dart @@ -14,26 +14,40 @@ class SystemConfigFFmpegRealtimeDto { /// Returns a new [SystemConfigFFmpegRealtimeDto] instance. SystemConfigFFmpegRealtimeDto({ required this.enabled, + this.resolutions = const [], + this.videoCodecs = const [], }); /// Enable real-time HLS transcoding (alpha) bool enabled; + /// Resolutions to use for real-time HLS transcoding + List resolutions; + + /// Video codecs to use for real-time HLS transcoding + List videoCodecs; + @override bool operator ==(Object other) => identical(this, other) || other is SystemConfigFFmpegRealtimeDto && - other.enabled == enabled; + other.enabled == enabled && + _deepEquality.equals(other.resolutions, resolutions) && + _deepEquality.equals(other.videoCodecs, videoCodecs); @override int get hashCode => // ignore: unnecessary_parenthesis - (enabled.hashCode); + (enabled.hashCode) + + (resolutions.hashCode) + + (videoCodecs.hashCode); @override - String toString() => 'SystemConfigFFmpegRealtimeDto[enabled=$enabled]'; + String toString() => 'SystemConfigFFmpegRealtimeDto[enabled=$enabled, resolutions=$resolutions, videoCodecs=$videoCodecs]'; Map toJson() { final json = {}; json[r'enabled'] = this.enabled; + json[r'resolutions'] = this.resolutions; + json[r'videoCodecs'] = this.videoCodecs; return json; } @@ -47,6 +61,8 @@ class SystemConfigFFmpegRealtimeDto { return SystemConfigFFmpegRealtimeDto( enabled: mapValueOfType(json, r'enabled')!, + resolutions: HlsVideoResolution.listFromJson(json[r'resolutions']), + videoCodecs: VideoCodec.listFromJson(json[r'videoCodecs']), ); } return null; @@ -95,6 +111,8 @@ class SystemConfigFFmpegRealtimeDto { /// The list of required keys that must be present in a JSON. static const requiredKeys = { 'enabled', + 'resolutions', + 'videoCodecs', }; } diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index e674f4712a..b1330fb55a 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -19120,6 +19120,17 @@ }, "type": "object" }, + "HlsVideoResolution": { + "description": "HLS video resolution", + "enum": [ + 480, + 720, + 1080, + 1440, + 2160 + ], + "type": "integer" + }, "ImageFormat": { "description": "Image format", "enum": [ @@ -25754,10 +25765,26 @@ "enabled": { "description": "Enable real-time HLS transcoding (alpha)", "type": "boolean" + }, + "resolutions": { + "description": "Resolutions to use for real-time HLS transcoding", + "items": { + "$ref": "#/components/schemas/HlsVideoResolution" + }, + "type": "array" + }, + "videoCodecs": { + "description": "Video codecs to use for real-time HLS transcoding", + "items": { + "$ref": "#/components/schemas/VideoCodec" + }, + "type": "array" } }, "required": [ - "enabled" + "enabled", + "resolutions", + "videoCodecs" ], "type": "object" }, diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts index 6413a26eb3..209a7b7f66 100644 --- a/packages/sdk/src/fetch-client.ts +++ b/packages/sdk/src/fetch-client.ts @@ -2302,6 +2302,10 @@ export type SystemConfigBackupsDto = { export type SystemConfigFFmpegRealtimeDto = { /** Enable real-time HLS transcoding (alpha) */ enabled: boolean; + /** Resolutions to use for real-time HLS transcoding */ + resolutions: HlsVideoResolution[]; + /** Video codecs to use for real-time HLS transcoding */ + videoCodecs: VideoCodec[]; }; export type SystemConfigFFmpegDto = { accel: TranscodeHWAccel; @@ -7629,6 +7633,13 @@ export enum CQMode { Cqp = "cqp", Icq = "icq" } +export enum HlsVideoResolution { + $480 = 480, + $720 = 720, + $1080 = 1080, + $1440 = 1440, + $2160 = 2160 +} export enum ToneMapping { Hable = "hable", Mobius = "mobius", diff --git a/server/src/config.ts b/server/src/config.ts index 723e7e564e..55304080a3 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -4,6 +4,7 @@ import { AudioCodec, Colorspace, CQMode, + HlsVideoResolution, ImageFormat, LogLevel, OAuthTokenEndpointAuthMethod, @@ -48,6 +49,8 @@ export type SystemConfig = { tonemap: ToneMapping; realtime: { enabled: boolean; + videoCodecs: VideoCodec[]; + resolutions: HlsVideoResolution[]; }; }; integrityChecks: { @@ -247,6 +250,8 @@ export const defaults = Object.freeze({ accelDecode: true, realtime: { enabled: false, + videoCodecs: [VideoCodec.H264, VideoCodec.Hevc], + resolutions: [HlsVideoResolution.p480, HlsVideoResolution.p720, HlsVideoResolution.p1080], }, }, integrityChecks: { diff --git a/server/src/constants.ts b/server/src/constants.ts index e0a6296e64..815a400ed8 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -235,14 +235,65 @@ export const HLS_PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl'; export const HLS_SEGMENT_DURATION = 2; export const HLS_SEGMENT_FILENAME_REGEX = /^seg_(\d+)\.m4s$/; export const HLS_VARIANTS = [ - { resolution: 480, codec: VideoCodec.Av1, bitrate: 1_000_000, codecString: 'av01.0.04M.08' }, - { resolution: 480, codec: VideoCodec.Hevc, bitrate: 1_200_000, codecString: 'hvc1.1.6.L90.B0' }, - { resolution: 480, codec: VideoCodec.H264, bitrate: 2_500_000, codecString: 'avc1.64001e' }, - { resolution: 720, codec: VideoCodec.Av1, bitrate: 2_000_000, codecString: 'av01.0.08M.08' }, - { resolution: 720, codec: VideoCodec.Hevc, bitrate: 2_500_000, codecString: 'hvc1.1.6.L93.B0' }, - { resolution: 720, codec: VideoCodec.H264, bitrate: 5_000_000, codecString: 'avc1.64001f' }, - { resolution: 1080, codec: VideoCodec.Av1, bitrate: 4_000_000, codecString: 'av01.0.09M.08' }, - { resolution: 1080, codec: VideoCodec.Hevc, bitrate: 4_500_000, codecString: 'hvc1.1.6.L120.B0' }, - { resolution: 1080, codec: VideoCodec.H264, bitrate: 8_000_000, codecString: 'avc1.640028' }, + { resolution: 480, codec: VideoCodec.Av1, bitrate: 1_000_000 }, + { resolution: 480, codec: VideoCodec.Hevc, bitrate: 1_200_000 }, + { resolution: 480, codec: VideoCodec.H264, bitrate: 2_500_000 }, + { resolution: 720, codec: VideoCodec.Av1, bitrate: 2_000_000 }, + { resolution: 720, codec: VideoCodec.Hevc, bitrate: 2_500_000 }, + { resolution: 720, codec: VideoCodec.H264, bitrate: 5_000_000 }, + { resolution: 1080, codec: VideoCodec.Av1, bitrate: 4_000_000 }, + { resolution: 1080, codec: VideoCodec.Hevc, bitrate: 4_500_000 }, + { resolution: 1080, codec: VideoCodec.H264, bitrate: 8_000_000 }, + { resolution: 1440, codec: VideoCodec.Av1, bitrate: 7_000_000 }, + { resolution: 1440, codec: VideoCodec.Hevc, bitrate: 8_000_000 }, + { resolution: 1440, codec: VideoCodec.H264, bitrate: 14_000_000 }, + { resolution: 2160, codec: VideoCodec.Av1, bitrate: 12_000_000 }, + { resolution: 2160, codec: VideoCodec.Hevc, bitrate: 14_000_000 }, + { resolution: 2160, codec: VideoCodec.H264, bitrate: 25_000_000 }, ]; export const HLS_VERSION = 7; + +export type CodecLevel = { maxFrame: number; maxRate: number; token: string }; + +// H.264 High profile: token is the hex level_idc. +export const H264_LEVELS: CodecLevel[] = [ + { maxFrame: 1620, maxRate: 40_500, token: '1e' }, // 3.0 + { maxFrame: 3600, maxRate: 108_000, token: '1f' }, // 3.1 + { maxFrame: 5120, maxRate: 216_000, token: '20' }, // 3.2 + { maxFrame: 8192, maxRate: 245_760, token: '28' }, // 4.0 + { maxFrame: 8704, maxRate: 522_240, token: '2a' }, // 4.2 + { maxFrame: 22_080, maxRate: 589_824, token: '32' }, // 5.0 + { maxFrame: 36_864, maxRate: 983_040, token: '33' }, // 5.1 + { maxFrame: 36_864, maxRate: 2_073_600, token: '34' }, // 5.2 + { maxFrame: 139_264, maxRate: 4_177_920, token: '3c' }, // 6.0 + { maxFrame: 139_264, maxRate: 8_355_840, token: '3d' }, // 6.1 + { maxFrame: 139_264, maxRate: 16_711_680, token: '3e' }, // 6.2 +]; + +// HEVC Main profile, Main tier: token is `L` + level_idc (level × 30). +export const HEVC_LEVELS: CodecLevel[] = [ + { maxFrame: 552_960, maxRate: 16_588_800, token: 'L90' }, // 3.0 + { maxFrame: 983_040, maxRate: 33_177_600, token: 'L93' }, // 3.1 + { maxFrame: 2_228_224, maxRate: 66_846_720, token: 'L120' }, // 4.0 + { maxFrame: 2_228_224, maxRate: 133_693_440, token: 'L123' }, // 4.1 + { maxFrame: 8_912_896, maxRate: 267_386_880, token: 'L150' }, // 5.0 + { maxFrame: 8_912_896, maxRate: 534_773_760, token: 'L153' }, // 5.1 + { maxFrame: 8_912_896, maxRate: 1_069_547_520, token: 'L156' }, // 5.2 + { maxFrame: 35_651_584, maxRate: 1_069_547_520, token: 'L180' }, // 6.0 + { maxFrame: 35_651_584, maxRate: 2_139_095_040, token: 'L183' }, // 6.1 + { maxFrame: 35_651_584, maxRate: 4_278_190_080, token: 'L186' }, // 6.2 +]; + +// AV1 Main profile (0), Main tier (M): token is the two-digit seq_level_idx + `M`. +export const AV1_LEVELS: CodecLevel[] = [ + { maxFrame: 665_856, maxRate: 19_975_168, token: '04M' }, // 3.0 + { maxFrame: 1_065_024, maxRate: 31_950_336, token: '05M' }, // 3.1 + { maxFrame: 2_359_296, maxRate: 70_778_880, token: '08M' }, // 4.0 + { maxFrame: 2_359_296, maxRate: 141_557_760, token: '09M' }, // 4.1 + { maxFrame: 8_912_896, maxRate: 267_386_880, token: '12M' }, // 5.0 + { maxFrame: 8_912_896, maxRate: 534_773_760, token: '13M' }, // 5.1 + { maxFrame: 8_912_896, maxRate: 1_069_547_520, token: '14M' }, // 5.2 + { maxFrame: 35_651_584, maxRate: 1_069_547_520, token: '16M' }, // 6.0 + { maxFrame: 35_651_584, maxRate: 2_139_095_040, token: '17M' }, // 6.1 + { maxFrame: 35_651_584, maxRate: 4_278_190_080, token: '18M' }, // 6.2 +]; diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 85618723b2..2569e1a52e 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -11,6 +11,7 @@ import { AudioCodecSchema, ColorspaceSchema, CQModeSchema, + HlsVideoResolutionSchema, ImageFormatSchema, LogLevelSchema, OAuthTokenEndpointAuthMethodSchema, @@ -115,6 +116,8 @@ const SystemConfigFFmpegSchema = z realtime: z .object({ enabled: configBool.describe('Enable real-time HLS transcoding (alpha)'), + videoCodecs: z.array(VideoCodecSchema).describe('Video codecs to use for real-time HLS transcoding'), + resolutions: z.array(HlsVideoResolutionSchema).describe('Resolutions to use for real-time HLS transcoding'), }) .meta({ id: 'SystemConfigFFmpegRealtimeDto' }), }) diff --git a/server/src/enum.ts b/server/src/enum.ts index 491beca10b..0996abe6fc 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -529,6 +529,19 @@ export enum CQMode { export const CQModeSchema = z.enum(CQMode).describe('CQ mode').meta({ id: 'CQMode' }); +export enum HlsVideoResolution { + p480 = 480, + p720 = 720, + p1080 = 1080, + p1440 = 1440, + p2160 = 2160, +} + +export const HlsVideoResolutionSchema = z + .enum(HlsVideoResolution) + .describe('HLS video resolution') + .meta({ id: 'HlsVideoResolution', type: 'integer' }); + export enum Colorspace { Srgb = 'srgb', P3 = 'p3', diff --git a/server/src/services/hls.service.spec.ts b/server/src/services/hls.service.spec.ts index a18f7c6d22..adfc3d5bcc 100644 --- a/server/src/services/hls.service.spec.ts +++ b/server/src/services/hls.service.spec.ts @@ -1,5 +1,5 @@ import { BadRequestException, NotFoundException } from '@nestjs/common'; -import { TranscodeHardwareAcceleration } from 'src/enum'; +import { HlsVideoResolution, VideoCodec } from 'src/enum'; import { HlsService } from 'src/services/hls.service'; import { eiffelTower, train, waterfall } from 'test/fixtures/media.stub'; import { factory } from 'test/small.factory'; @@ -96,67 +96,79 @@ seg_10.m4s const sessionId = '00000000-0000-0000-0000-000000000000'; -const eiffelExpectedMasterDisabled = `#EXTM3U +const eiffelExpectedMasterAv1 = `#EXTM3U #EXT-X-VERSION:7 #EXT-X-INDEPENDENT-SEGMENTS -#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=1350000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/0/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=1620000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/1/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=3375000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/2/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=2700000,RESOLUTION=720x1280,CODECS="av01.0.05M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/3/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=3375000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/4/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=6750000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/5/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=5400000,RESOLUTION=1080x1920,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/6/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=6075000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/7/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=10800000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/8/playlist.m3u8 `; -const eiffelExpectedMasterRkmpp = `#EXTM3U +const eiffelExpectedMasterNoAv1 = `#EXTM3U #EXT-X-VERSION:7 #EXT-X-INDEPENDENT-SEGMENTS -#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=1620000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/1/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=3375000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/2/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=3375000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/4/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=6750000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/5/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=6075000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/7/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +#EXT-X-STREAM-INF:BANDWIDTH=10800000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/8/playlist.m3u8 `; -const waterfallExpectedMasterDisabled = `#EXTM3U +const waterfallExpectedMasterAv1 = `#EXTM3U #EXT-X-VERSION:7 #EXT-X-INDEPENDENT-SEGMENTS -#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=1350000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/0/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=1620000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/1/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=3375000,RESOLUTION=480x852,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/2/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=2700000,RESOLUTION=720x1280,CODECS="av01.0.05M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/3/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=3375000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/4/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=6750000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/5/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=5400000,RESOLUTION=1080x1920,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/6/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=6075000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/7/playlist.m3u8 -#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +#EXT-X-STREAM-INF:BANDWIDTH=10800000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/8/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=9450000,RESOLUTION=1440x2560,CODECS="av01.0.12M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/9/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=10800000,RESOLUTION=1440x2560,CODECS="hvc1.1.6.L150.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/10/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=18900000,RESOLUTION=1440x2560,CODECS="avc1.640032,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/11/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=16200000,RESOLUTION=2160x3840,CODECS="av01.0.12M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/12/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=18900000,RESOLUTION=2160x3840,CODECS="hvc1.1.6.L150.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/13/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=33750000,RESOLUTION=2160x3840,CODECS="avc1.640033,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/14/playlist.m3u8 `; describe(HlsService.name, () => { @@ -171,9 +183,24 @@ describe(HlsService.name, () => { const auth = factory.auth(); const assetId = 'asset-1'; - const setup = (asset: typeof eiffelTower | typeof waterfall, accel: TranscodeHardwareAcceleration) => { + const allCodecs = [VideoCodec.Av1, VideoCodec.Hevc, VideoCodec.H264]; + const allResolutions = [ + HlsVideoResolution.p480, + HlsVideoResolution.p720, + HlsVideoResolution.p1080, + HlsVideoResolution.p1440, + HlsVideoResolution.p2160, + ]; + + const setup = ( + asset: typeof eiffelTower | typeof waterfall, + videoCodecs?: VideoCodec[], + resolutions?: HlsVideoResolution[], + ) => { mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); - mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true }, accel } }); + mocks.systemMetadata.get.mockResolvedValue({ + ffmpeg: { realtime: { enabled: true, videoCodecs, resolutions } }, + }); mocks.videoStream.getForMainPlaylist.mockResolvedValue(asset); mocks.crypto.randomUUID.mockReturnValue(sessionId); mocks.websocket.serverSend.mockImplementation((event, ...rest) => { @@ -184,19 +211,19 @@ describe(HlsService.name, () => { }); }; - it('returns main playlist for eiffel-tower (1080p portrait, no acceleration)', async () => { - setup(eiffelTower, TranscodeHardwareAcceleration.Disabled); - await expect(sut.getMainPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterDisabled); + it('offers AV1, HEVC, and H.264 when AV1 is configured and the accelerator supports it', async () => { + setup(eiffelTower, allCodecs); + await expect(sut.getMainPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterAv1); }); - it('returns main playlist for eiffel-tower with RKMPP (no AV1 variants)', async () => { - setup(eiffelTower, TranscodeHardwareAcceleration.Rkmpp); - await expect(sut.getMainPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterRkmpp); + it('omits AV1 when it is not in the configured codecs', async () => { + setup(eiffelTower); + await expect(sut.getMainPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterNoAv1); }); - it('returns main playlist for waterfall (4K landscape) with no acceleration', async () => { - setup(waterfall, TranscodeHardwareAcceleration.Disabled); - await expect(sut.getMainPlaylist(auth, assetId)).resolves.toBe(waterfallExpectedMasterDisabled); + it('offers every resolution up to the source and derives 4K codec levels (waterfall, 4K, 29.83fps)', async () => { + setup(waterfall, allCodecs, allResolutions); + await expect(sut.getMainPlaylist(auth, assetId)).resolves.toBe(waterfallExpectedMasterAv1); }); it('throws BadRequestException when realtime transcoding is disabled', async () => { diff --git a/server/src/services/hls.service.ts b/server/src/services/hls.service.ts index 0cbb53e1cd..94a8a46ace 100644 --- a/server/src/services/hls.service.ts +++ b/server/src/services/hls.service.ts @@ -1,13 +1,7 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { constants } from 'node:fs'; import { join } from 'node:path'; -import { - HLS_SEGMENT_DURATION, - HLS_SEGMENT_FILENAME_REGEX, - HLS_VARIANTS, - HLS_VERSION, - SUPPORTED_HWA_CODECS, -} from 'src/constants'; +import { HLS_SEGMENT_DURATION, HLS_SEGMENT_FILENAME_REGEX, HLS_VARIANTS, HLS_VERSION } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; @@ -18,7 +12,7 @@ import { BaseService } from 'src/services/base.service'; import { VideoPacketInfo, VideoStreamInfo } from 'src/types'; import { PendingEvents } from 'src/utils/event'; import { ImmichFileResponse } from 'src/utils/file'; -import { getOutputSize } from 'src/utils/media'; +import { getCodecString, getOutputSize } from 'src/utils/media'; type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo }; type Segmentation = { fps: number; framesPerSegment: number; segmentCount: number; segmentDuration: number }; @@ -131,18 +125,21 @@ export class HlsService extends BaseService { } private generateMainPlaylist(sessionId: string, ffmpeg: SystemConfigFFmpegDto, asset: AssetWithStreamInfo) { - const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3); + const fps = (asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration; + const roundedFps = fps.toFixed(3); const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width); const targetResolution = Math.max(sourceResolution, HLS_VARIANTS[0].resolution); const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`, '#EXT-X-INDEPENDENT-SEGMENTS']; + const { videoCodecs, resolutions } = ffmpeg.realtime; for (let i = 0; i < HLS_VARIANTS.length; i++) { - const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i]; - if (resolution > targetResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) { + const { resolution, bitrate, codec } = HLS_VARIANTS[i]; + if (resolution > targetResolution || !videoCodecs.includes(codec) || !resolutions.includes(resolution)) { continue; } const { width, height } = getOutputSize(asset.videoStream, resolution); + const codecString = getCodecString(codec, width, height, fps); lines.push( - `#EXT-X-STREAM-INF:BANDWIDTH=${bitrate},RESOLUTION=${width}x${height},CODECS="${codecString},mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=${fps}`, + `#EXT-X-STREAM-INF:BANDWIDTH=${Math.round(bitrate * 1.35)},RESOLUTION=${width}x${height},CODECS="${codecString},mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=${roundedFps}`, `${sessionId}/${i}/playlist.m3u8`, ); } diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 920b9b9346..d5404048c0 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -5,6 +5,7 @@ import { AudioCodec, Colorspace, CQMode, + HlsVideoResolution, ImageFormat, LogLevel, OAuthTokenEndpointAuthMethod, @@ -76,6 +77,8 @@ const updatedConfig = Object.freeze({ tonemap: ToneMapping.Hable, realtime: { enabled: false, + videoCodecs: [VideoCodec.H264, VideoCodec.Hevc], + resolutions: [HlsVideoResolution.p480, HlsVideoResolution.p720, HlsVideoResolution.p1080], }, }, integrityChecks: { diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index de925cf571..5c81001eba 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -1,4 +1,4 @@ -import { AUDIO_ENCODER, SUPPORTED_HWA_CODECS } from 'src/constants'; +import { AUDIO_ENCODER, AV1_LEVELS, CodecLevel, H264_LEVELS, HEVC_LEVELS, SUPPORTED_HWA_CODECS } from 'src/constants'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; import { ColorMatrix, @@ -36,6 +36,29 @@ export const getOutputSize = (videoStream: VideoStreamInfo, targetRes: number) = return isVideoVertical(videoStream) ? { width: targetRes, height: larger } : { width: larger, height: targetRes }; }; +const pickLevel = (levels: CodecLevel[], frame: number, rate: number) => + levels.find((level) => frame <= level.maxFrame && rate <= level.maxRate) ?? levels.at(-1)!; + +export const getCodecString = (codec: VideoCodec, width: number, height: number, fps: number): string => { + switch (codec) { + case VideoCodec.H264: { + const macroblocks = Math.ceil(width / 16) * Math.ceil(height / 16); + return `avc1.6400${pickLevel(H264_LEVELS, macroblocks, macroblocks * fps).token}`; + } + case VideoCodec.Hevc: { + const samples = width * height; + return `hvc1.1.6.${pickLevel(HEVC_LEVELS, samples, samples * fps).token}.B0`; + } + case VideoCodec.Av1: { + const samples = width * height; + return `av01.0.${pickLevel(AV1_LEVELS, samples, samples * fps).token}.08`; + } + default: { + throw new Error(`Codec '${codec}' does not support HLS codec strings`); + } + } +}; + export class BaseConfig implements VideoCodecSWConfig { readonly presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast']; protected constructor( diff --git a/web/src/routes/admin/system-settings/FFmpegSettings.svelte b/web/src/routes/admin/system-settings/FFmpegSettings.svelte index 101de86008..2e173eaa7e 100644 --- a/web/src/routes/admin/system-settings/FFmpegSettings.svelte +++ b/web/src/routes/admin/system-settings/FFmpegSettings.svelte @@ -12,6 +12,7 @@ import { AudioCodec, CQMode, + HlsVideoResolution, ToneMapping, TranscodeHWAccel, TranscodePolicy, @@ -401,9 +402,45 @@ title={$t('admin.transcoding_realtime_enabled')} subtitle={$t('admin.transcoding_realtime_enabled_description')} bind:checked={configToEdit.ffmpeg.realtime.enabled} - isEdited={configToEdit.ffmpeg.realtime.enabled !== configToEdit.ffmpeg.realtime.enabled} + isEdited={configToEdit.ffmpeg.realtime.enabled !== config.ffmpeg.realtime.enabled} {disabled} /> + + + + diff --git a/web/src/routes/admin/system-settings/SettingCheckboxes.svelte b/web/src/routes/admin/system-settings/SettingCheckboxes.svelte index d085482d27..5676ab6584 100644 --- a/web/src/routes/admin/system-settings/SettingCheckboxes.svelte +++ b/web/src/routes/admin/system-settings/SettingCheckboxes.svelte @@ -1,4 +1,4 @@ -