From 1a07f335939a9c58a3dbbbef9dd1c8651b64ffa2 Mon Sep 17 00:00:00 2001 From: bwees Date: Wed, 11 Mar 2026 10:16:47 -0500 Subject: [PATCH] feat: wip --- open-api/immich-openapi-specs.json | 1 + server/src/enum.ts | 1 + .../src/repositories/asset-job.repository.ts | 1 + server/src/repositories/media.repository.ts | 21 ++-- server/src/services/asset.service.ts | 8 +- server/src/services/media.service.ts | 47 ++++++++- server/src/types.ts | 5 +- server/src/utils/media.ts | 98 ++++++++++++++++--- web/src/lib/services/asset.service.ts | 1 - 9 files changed, 156 insertions(+), 27 deletions(-) diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index d2eb322009..c70ba24231 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -18160,6 +18160,7 @@ "AssetDetectDuplicatesQueueAll", "AssetDetectDuplicates", "AssetEditThumbnailGeneration", + "AssetEditTranscodeGeneration", "AssetEncodeVideoQueueAll", "AssetEncodeVideo", "AssetEmptyTrash", diff --git a/server/src/enum.ts b/server/src/enum.ts index 60f45efd6e..de0e106842 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -589,6 +589,7 @@ export enum JobName { AssetDetectDuplicatesQueueAll = 'AssetDetectDuplicatesQueueAll', AssetDetectDuplicates = 'AssetDetectDuplicates', AssetEditThumbnailGeneration = 'AssetEditThumbnailGeneration', + AssetEditTranscodeGeneration = 'AssetEditTranscodeGeneration', AssetEncodeVideoQueueAll = 'AssetEncodeVideoQueueAll', AssetEncodeVideo = 'AssetEncodeVideo', AssetEmptyTrash = 'AssetEmptyTrash', diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 3765cad7ed..e503ab8dd0 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -335,6 +335,7 @@ export class AssetJobRepository { .selectFrom('asset') .select(['asset.id', 'asset.ownerId', 'asset.originalPath']) .select(withFiles) + .select(withEdits) .where('asset.id', '=', id) .where('asset.type', '=', sql.lit(AssetType.Video)) .executeTakeFirst(); diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 58e006171a..207d4e5358 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -29,7 +29,7 @@ const probe = (input: string, options: string[]): Promise => sharp.concurrency(0); sharp.cache({ files: 0 }); -type ProgressEvent = { +export type ProgressEvent = { frames: number; currentFps: number; currentKbps: number; @@ -327,7 +327,7 @@ export class MediaRepository { const { frameCount, percentInterval } = options.progress; const frameInterval = Math.ceil(frameCount / (100 / percentInterval)); - if (this.logger.isLevelEnabled(LogLevel.Debug) && frameCount && frameInterval) { + if (frameCount && frameInterval) { let lastProgressFrame: number = 0; ffmpegCall.on('progress', (progress: ProgressEvent) => { if (progress.frames - lastProgressFrame < frameInterval) { @@ -336,12 +336,17 @@ export class MediaRepository { lastProgressFrame = progress.frames; const percent = ((progress.frames / frameCount) * 100).toFixed(2); - const ms = progress.currentFps ? Math.floor((frameCount - progress.frames) / progress.currentFps) * 1000 : 0; - const duration = ms ? Duration.fromMillis(ms).rescale().toHuman({ unitDisplay: 'narrow' }) : ''; - const outputText = output instanceof Writable ? 'stream' : output.split('/').pop(); - this.logger.debug( - `Transcoding ${percent}% done${duration ? `, estimated ${duration} remaining` : ''} for output ${outputText}`, - ); + + options.progress.callback?.(progress.frames / frameCount, progress.frames); + + if (this.logger.isLevelEnabled(LogLevel.Debug)) { + const ms = progress.currentFps ? Math.floor((frameCount - progress.frames) / progress.currentFps) * 1000 : 0; + const duration = ms ? Duration.fromMillis(ms).rescale().toHuman({ unitDisplay: 'narrow' }) : ''; + const outputText = output instanceof Writable ? 'stream' : output.split('/').pop(); + this.logger.debug( + `Transcoding ${percent}% done${duration ? `, estimated ${duration} remaining` : ''} for output ${outputText}`, + ); + } }); } diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index 1e5d23a98d..bd8df37afb 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -565,10 +565,6 @@ export class AssetService extends BaseService { throw new BadRequestException('Only images can be edited'); } - if (asset.livePhotoVideoId) { - throw new BadRequestException('Editing live photos is not supported'); - } - if (isPanorama(asset)) { throw new BadRequestException('Editing panorama images is not supported'); } @@ -611,6 +607,10 @@ export class AssetService extends BaseService { const newEdits = await this.assetEditRepository.replaceAll(id, edits); await this.jobRepository.queue({ name: JobName.AssetEditThumbnailGeneration, data: { id } }); + if (asset.livePhotoVideoId) { + await this.jobRepository.queue({ name: JobName.AssetEditTranscodeGeneration, data: { id } }); + } + // Return the asset and its applied edits return { assetId: id, diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index ea0b1e9142..8bfc8c7800 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -207,6 +207,51 @@ export class MediaService extends BaseService { return JobStatus.Success; } + @OnJob({ name: JobName.AssetEditTranscodeGeneration, queue: QueueName.Editor }) + async handleAssetEditTranscodeGeneration({ id }: JobOf): Promise { + const asset = await this.assetJobRepository.getForVideoConversion(id); + if (!asset) { + return JobStatus.Failed; + } + + const input = asset.originalPath; + const output = StorageCore.getEncodedVideoPath(asset); + this.storageCore.ensureFolders(output); + + const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, { + countFrames: this.logger.isLevelEnabled(LogLevel.Debug), // makes frame count more reliable for progress logs + }); + const videoStream = this.getMainStream(videoStreams); + const audioStream = this.getMainStream(audioStreams); + if (!videoStream || !format.formatName) { + return JobStatus.Failed; + } + + if (!videoStream.height || !videoStream.width) { + this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video streams found`); + return JobStatus.Failed; + } + + let { ffmpeg } = await this.getConfig({ withCache: true }); + ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; + const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand( + TranscodeTarget.All, + videoStream, + audioStream, + undefined, // TODO: cleaner way to do this? + asset.edits, + ); + await this.mediaRepository.transcode(input, output, command); + + this.logger.log(`Successfully encoded ${asset.id}`); + console.log(`Successfully encoded ${asset.id}`); + console.log(`New Path: ${output}`); + + await this.assetRepository.update({ id: asset.id, encodedVideoPath: output }); + + return JobStatus.Success; + } + @OnJob({ name: JobName.AssetGenerateThumbnails, queue: QueueName.ThumbnailGeneration }) async handleGenerateThumbnails({ id }: JobOf): Promise { const asset = await this.assetJobRepository.getForGenerateThumbnailJob(id); @@ -649,7 +694,7 @@ export class MediaService extends BaseService { if (!partialFallbackSuccess) { this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled`); - ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; + ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; // TODO: USE THIS TO DISABLE CPU ENCODING const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand(target, videoStream, audioStream); await this.mediaRepository.transcode(input, output, command); } diff --git a/server/src/types.ts b/server/src/types.ts index 33174e187e..5b6d6a0bda 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -130,6 +130,7 @@ export interface TranscodeCommand { progress: { frameCount: number; percentInterval: number; + callback: (percent: number, frame: number) => void; }; } @@ -151,6 +152,7 @@ export interface VideoCodecSWConfig { videoStream: VideoStreamInfo, audioStream: AudioStreamInfo, format?: VideoFormat, + edits?: AssetEditActionItem[], ): TranscodeCommand; } @@ -389,7 +391,8 @@ export type JobItem = | { name: JobName.WorkflowRun; data: IWorkflowJob } // Editor - | { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob }; + | { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob } + | { name: JobName.AssetEditTranscodeGeneration; data: IEntityJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index ce185305bd..039de27ce2 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -1,6 +1,14 @@ import { AUDIO_ENCODER } from 'src/constants'; +import { + AssetEditAction, + AssetEditActionItem, + CropParameters, + MirrorAxis, + MirrorParameters, + RotateParameters, +} from 'src/dtos/editing.dto'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; -import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum'; +import { CQMode, LogLevel, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum'; import { AudioStreamInfo, BitrateDistribution, @@ -88,6 +96,7 @@ export class BaseConfig implements VideoCodecSWConfig { videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo, format?: VideoFormat, + edits: AssetEditActionItem[] = [], ) { const options = { inputOptions: this.getBaseInputOptions(videoStream, format), @@ -95,8 +104,9 @@ export class BaseConfig implements VideoCodecSWConfig { twoPass: this.eligibleForTwoPass(), progress: { frameCount: videoStream.frameCount, percentInterval: 5 }, } as TranscodeCommand; + if ([TranscodeTarget.All, TranscodeTarget.Video].includes(target)) { - const filters = this.getFilterOptions(videoStream); + const filters = this.getFilterOptions(videoStream, edits); if (filters.length > 0) { options.outputOptions.push(`-vf ${filters.join(',')}`); } @@ -156,10 +166,46 @@ export class BaseConfig implements VideoCodecSWConfig { return options; } - getFilterOptions(videoStream: VideoStreamInfo) { + getEditOptions(videoStream: VideoStreamInfo, edits: AssetEditActionItem[]) { const options = []; - if (this.shouldScale(videoStream)) { - options.push(`scale=${this.getScaling(videoStream)}`); + let currentDimensions = { width: videoStream.width, height: videoStream.height }; + + // Apply CPU edit operations before hwupload + for (const edit of edits) { + switch (edit.action) { + case AssetEditAction.Crop: { + options.push(this.getCropOperation(edit.parameters)); + currentDimensions = { width: edit.parameters.width, height: edit.parameters.height }; + break; + } + case AssetEditAction.Rotate: { + const rotateFilter = this.getRotateOperation(edit.parameters); + if (rotateFilter) { + options.push(rotateFilter); + if (Math.abs(edit.parameters.angle) === 90 || Math.abs(edit.parameters.angle) === 270) { + currentDimensions = { width: currentDimensions.height, height: currentDimensions.width }; + } + } + break; + } + case AssetEditAction.Mirror: { + options.push(this.getMirrorOperation(edit.parameters)); + break; + } + } + } + + return { options, currentDimensions }; + } + + getFilterOptions(videoStream: VideoStreamInfo, edits: AssetEditActionItem[] = []) { + const options = []; + const { options: editOptions, currentDimensions } = this.getEditOptions(videoStream, edits); + options.push(...editOptions); + + // Apply scaling based on current dimensions after edits + if (this.shouldScale(videoStream, currentDimensions)) { + options.push(`scale=${this.getScaling(videoStream, 2, currentDimensions)}`); } const tonemapOptions = this.getToneMapping(videoStream); @@ -238,9 +284,10 @@ export class BaseConfig implements VideoCodecSWConfig { return target; } - shouldScale(videoStream: VideoStreamInfo) { - const oddDimensions = videoStream.height % 2 !== 0 || videoStream.width % 2 !== 0; - const largerThanTarget = Math.min(videoStream.height, videoStream.width) > this.getTargetResolution(videoStream); + shouldScale(videoStream: VideoStreamInfo, currentDimensions?: { width: number; height: number }) { + const dims = currentDimensions || { width: videoStream.width, height: videoStream.height }; + const oddDimensions = dims.height % 2 !== 0 || dims.width % 2 !== 0; + const largerThanTarget = Math.min(dims.height, dims.width) > this.getTargetResolution(videoStream); return oddDimensions || largerThanTarget; } @@ -248,9 +295,11 @@ export class BaseConfig implements VideoCodecSWConfig { return videoStream.isHDR && this.config.tonemap !== ToneMapping.Disabled; } - getScaling(videoStream: VideoStreamInfo, mult = 2) { + getScaling(videoStream: VideoStreamInfo, mult = 2, currentDimensions?: { width: number; height: number }) { + const dims = currentDimensions || { width: videoStream.width, height: videoStream.height }; const targetResolution = this.getTargetResolution(videoStream); - return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; + const isVertical = dims.height > dims.width || this.isVideoRotated(videoStream); + return isVertical ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; } getSize(videoStream: VideoStreamInfo) { @@ -329,6 +378,31 @@ export class BaseConfig implements VideoCodecSWConfig { useCQP() { return this.config.cqMode === CQMode.Cqp; } + + // Edit operations (software filters) + getCropOperation({ x, y, width, height }: CropParameters): string { + return `crop=${width}:${height}:${x}:${y}`; + } + + getRotateOperation({ angle }: RotateParameters): string { + switch (angle) { + case 90: { + return 'transpose=1'; // 90° clockwise + } + case 180: { + return 'hflip,vflip'; // 180° + } + case 270: { + return 'transpose=2'; // 90° counter-clockwise (270° clockwise) + } + } + + return ''; + } + + getMirrorOperation({ axis }: MirrorParameters): string { + return axis === MirrorAxis.Horizontal ? 'hflip' : 'vflip'; + } } export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { @@ -423,14 +497,14 @@ export class ThumbnailConfig extends BaseConfig { return ['-fps_mode vfr', '-frames:v 1', '-update 1']; } - getFilterOptions(videoStream: VideoStreamInfo): string[] { + getFilterOptions(videoStream: VideoStreamInfo, edits: AssetEditActionItem[] = []): string[] { return [ 'fps=12:start_time=0:eof_action=pass:round=down', 'thumbnail=12', String.raw`select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20)`, 'trim=end_frame=2', 'reverse', - ...super.getFilterOptions(videoStream), + ...super.getFilterOptions(videoStream, edits), ]; } diff --git a/web/src/lib/services/asset.service.ts b/web/src/lib/services/asset.service.ts index 5d7ae07684..53869d86c6 100644 --- a/web/src/lib/services/asset.service.ts +++ b/web/src/lib/services/asset.service.ts @@ -243,7 +243,6 @@ export const getAssetActions = ($t: MessageFormatter, asset: AssetResponseDto) = !sharedLink && isOwner && asset.type === AssetTypeEnum.Image && - !asset.livePhotoVideoId && asset.exifInfo?.projectionType !== ProjectionType.EQUIRECTANGULAR && !asset.originalPath.toLowerCase().endsWith('.insp') && !asset.originalPath.toLowerCase().endsWith('.gif') &&