feat: wip

open-api/immich-openapi-specs.json

@@ -18160,6 +18160,7 @@
           "AssetDetectDuplicatesQueueAll",
           "AssetDetectDuplicates",
           "AssetEditThumbnailGeneration",
+          "AssetEditTranscodeGeneration",
           "AssetEncodeVideoQueueAll",
           "AssetEncodeVideo",
           "AssetEmptyTrash",

server/src/enum.ts

@@ -589,6 +589,7 @@ export enum JobName {
   AssetDetectDuplicatesQueueAll = 'AssetDetectDuplicatesQueueAll',
   AssetDetectDuplicates = 'AssetDetectDuplicates',
   AssetEditThumbnailGeneration = 'AssetEditThumbnailGeneration',
+  AssetEditTranscodeGeneration = 'AssetEditTranscodeGeneration',
   AssetEncodeVideoQueueAll = 'AssetEncodeVideoQueueAll',
   AssetEncodeVideo = 'AssetEncodeVideo',
   AssetEmptyTrash = 'AssetEmptyTrash',

server/src/repositories/asset-job.repository.ts

@@ -335,6 +335,7 @@ export class AssetJobRepository {
       .selectFrom('asset')
       .select(['asset.id', 'asset.ownerId', 'asset.originalPath'])
       .select(withFiles)
+      .select(withEdits)
       .where('asset.id', '=', id)
       .where('asset.type', '=', sql.lit(AssetType.Video))
       .executeTakeFirst();

server/src/repositories/media.repository.ts

@@ -29,7 +29,7 @@ const probe = (input: string, options: string[]): Promise<FfprobeData> =>
 sharp.concurrency(0);
 sharp.cache({ files: 0 });
 
-type ProgressEvent = {
+export type ProgressEvent = {
   frames: number;
   currentFps: number;
   currentKbps: number;
@@ -327,7 +327,7 @@ export class MediaRepository {
 
     const { frameCount, percentInterval } = options.progress;
     const frameInterval = Math.ceil(frameCount / (100 / percentInterval));
-    if (this.logger.isLevelEnabled(LogLevel.Debug) && frameCount && frameInterval) {
+    if (frameCount && frameInterval) {
       let lastProgressFrame: number = 0;
       ffmpegCall.on('progress', (progress: ProgressEvent) => {
         if (progress.frames - lastProgressFrame < frameInterval) {
@@ -336,12 +336,17 @@ export class MediaRepository {
 
         lastProgressFrame = progress.frames;
         const percent = ((progress.frames / frameCount) * 100).toFixed(2);
-        const ms = progress.currentFps ? Math.floor((frameCount - progress.frames) / progress.currentFps) * 1000 : 0;
+
-        const duration = ms ? Duration.fromMillis(ms).rescale().toHuman({ unitDisplay: 'narrow' }) : '';
+        options.progress.callback?.(progress.frames / frameCount, progress.frames);
-        const outputText = output instanceof Writable ? 'stream' : output.split('/').pop();
+
-        this.logger.debug(
+        if (this.logger.isLevelEnabled(LogLevel.Debug)) {
-          `Transcoding ${percent}% done${duration ? `, estimated ${duration} remaining` : ''} for output ${outputText}`,
+          const ms = progress.currentFps ? Math.floor((frameCount - progress.frames) / progress.currentFps) * 1000 : 0;
-        );
+          const duration = ms ? Duration.fromMillis(ms).rescale().toHuman({ unitDisplay: 'narrow' }) : '';
+          const outputText = output instanceof Writable ? 'stream' : output.split('/').pop();
+          this.logger.debug(
+            `Transcoding ${percent}% done${duration ? `, estimated ${duration} remaining` : ''} for output ${outputText}`,
+          );
+        }
       });
     }
 

server/src/services/asset.service.ts

@@ -565,10 +565,6 @@ export class AssetService extends BaseService {
       throw new BadRequestException('Only images can be edited');
     }
 
-    if (asset.livePhotoVideoId) {
-      throw new BadRequestException('Editing live photos is not supported');
-    }
-
     if (isPanorama(asset)) {
       throw new BadRequestException('Editing panorama images is not supported');
     }
@@ -611,6 +607,10 @@ export class AssetService extends BaseService {
     const newEdits = await this.assetEditRepository.replaceAll(id, edits);
     await this.jobRepository.queue({ name: JobName.AssetEditThumbnailGeneration, data: { id } });
 
+    if (asset.livePhotoVideoId) {
+      await this.jobRepository.queue({ name: JobName.AssetEditTranscodeGeneration, data: { id } });
+    }
+
     // Return the asset and its applied edits
     return {
       assetId: id,

server/src/services/media.service.ts

@@ -207,6 +207,51 @@ export class MediaService extends BaseService {
     return JobStatus.Success;
   }
 
+  @OnJob({ name: JobName.AssetEditTranscodeGeneration, queue: QueueName.Editor })
+  async handleAssetEditTranscodeGeneration({ id }: JobOf<JobName.AssetEditTranscodeGeneration>): Promise<JobStatus> {
+    const asset = await this.assetJobRepository.getForVideoConversion(id);
+    if (!asset) {
+      return JobStatus.Failed;
+    }
+
+    const input = asset.originalPath;
+    const output = StorageCore.getEncodedVideoPath(asset);
+    this.storageCore.ensureFolders(output);
+
+    const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, {
+      countFrames: this.logger.isLevelEnabled(LogLevel.Debug), // makes frame count more reliable for progress logs
+    });
+    const videoStream = this.getMainStream(videoStreams);
+    const audioStream = this.getMainStream(audioStreams);
+    if (!videoStream || !format.formatName) {
+      return JobStatus.Failed;
+    }
+
+    if (!videoStream.height || !videoStream.width) {
+      this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video streams found`);
+      return JobStatus.Failed;
+    }
+
+    let { ffmpeg } = await this.getConfig({ withCache: true });
+    ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled };
+    const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand(
+      TranscodeTarget.All,
+      videoStream,
+      audioStream,
+      undefined, // TODO: cleaner way to do this?
+      asset.edits,
+    );
+    await this.mediaRepository.transcode(input, output, command);
+
+    this.logger.log(`Successfully encoded ${asset.id}`);
+    console.log(`Successfully encoded ${asset.id}`);
+    console.log(`New Path: ${output}`);
+
+    await this.assetRepository.update({ id: asset.id, encodedVideoPath: output });
+
+    return JobStatus.Success;
+  }
+
   @OnJob({ name: JobName.AssetGenerateThumbnails, queue: QueueName.ThumbnailGeneration })
   async handleGenerateThumbnails({ id }: JobOf<JobName.AssetGenerateThumbnails>): Promise<JobStatus> {
     const asset = await this.assetJobRepository.getForGenerateThumbnailJob(id);
@@ -649,7 +694,7 @@ export class MediaService extends BaseService {
 
       if (!partialFallbackSuccess) {
         this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled`);
-        ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled };
+        ffmpeg = { ...ffmpeg, accel: TranscodeHardwareAcceleration.Disabled }; // TODO: USE THIS TO DISABLE CPU ENCODING
         const command = BaseConfig.create(ffmpeg, this.videoInterfaces).getCommand(target, videoStream, audioStream);
         await this.mediaRepository.transcode(input, output, command);
       }

server/src/types.ts

@@ -130,6 +130,7 @@ export interface TranscodeCommand {
   progress: {
     frameCount: number;
     percentInterval: number;
+    callback: (percent: number, frame: number) => void;
   };
 }
 
@@ -151,6 +152,7 @@ export interface VideoCodecSWConfig {
     videoStream: VideoStreamInfo,
     audioStream: AudioStreamInfo,
     format?: VideoFormat,
+    edits?: AssetEditActionItem[],
   ): TranscodeCommand;
 }
 
@@ -389,7 +391,8 @@ export type JobItem =
   | { name: JobName.WorkflowRun; data: IWorkflowJob }
 
   // Editor
-  | { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob };
+  | { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob }
+  | { name: JobName.AssetEditTranscodeGeneration; data: IEntityJob };
 
 export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];
 

server/src/utils/media.ts

@@ -1,6 +1,14 @@
 import { AUDIO_ENCODER } from 'src/constants';
+import {
+  AssetEditAction,
+  AssetEditActionItem,
+  CropParameters,
+  MirrorAxis,
+  MirrorParameters,
+  RotateParameters,
+} from 'src/dtos/editing.dto';
 import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
-import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum';
+import { CQMode, LogLevel, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum';
 import {
   AudioStreamInfo,
   BitrateDistribution,
@@ -88,6 +96,7 @@ export class BaseConfig implements VideoCodecSWConfig {
     videoStream: VideoStreamInfo,
     audioStream?: AudioStreamInfo,
     format?: VideoFormat,
+    edits: AssetEditActionItem[] = [],
   ) {
     const options = {
       inputOptions: this.getBaseInputOptions(videoStream, format),
@@ -95,8 +104,9 @@ export class BaseConfig implements VideoCodecSWConfig {
       twoPass: this.eligibleForTwoPass(),
       progress: { frameCount: videoStream.frameCount, percentInterval: 5 },
     } as TranscodeCommand;
+
     if ([TranscodeTarget.All, TranscodeTarget.Video].includes(target)) {
-      const filters = this.getFilterOptions(videoStream);
+      const filters = this.getFilterOptions(videoStream, edits);
       if (filters.length > 0) {
         options.outputOptions.push(`-vf ${filters.join(',')}`);
       }
@@ -156,10 +166,46 @@ export class BaseConfig implements VideoCodecSWConfig {
     return options;
   }
 
-  getFilterOptions(videoStream: VideoStreamInfo) {
+  getEditOptions(videoStream: VideoStreamInfo, edits: AssetEditActionItem[]) {
     const options = [];
-    if (this.shouldScale(videoStream)) {
+    let currentDimensions = { width: videoStream.width, height: videoStream.height };
-      options.push(`scale=${this.getScaling(videoStream)}`);
+
+    // Apply CPU edit operations before hwupload
+    for (const edit of edits) {
+      switch (edit.action) {
+        case AssetEditAction.Crop: {
+          options.push(this.getCropOperation(edit.parameters));
+          currentDimensions = { width: edit.parameters.width, height: edit.parameters.height };
+          break;
+        }
+        case AssetEditAction.Rotate: {
+          const rotateFilter = this.getRotateOperation(edit.parameters);
+          if (rotateFilter) {
+            options.push(rotateFilter);
+            if (Math.abs(edit.parameters.angle) === 90 || Math.abs(edit.parameters.angle) === 270) {
+              currentDimensions = { width: currentDimensions.height, height: currentDimensions.width };
+            }
+          }
+          break;
+        }
+        case AssetEditAction.Mirror: {
+          options.push(this.getMirrorOperation(edit.parameters));
+          break;
+        }
+      }
+    }
+
+    return { options, currentDimensions };
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo, edits: AssetEditActionItem[] = []) {
+    const options = [];
+    const { options: editOptions, currentDimensions } = this.getEditOptions(videoStream, edits);
+    options.push(...editOptions);
+
+    // Apply scaling based on current dimensions after edits
+    if (this.shouldScale(videoStream, currentDimensions)) {
+      options.push(`scale=${this.getScaling(videoStream, 2, currentDimensions)}`);
     }
 
     const tonemapOptions = this.getToneMapping(videoStream);
@@ -238,9 +284,10 @@ export class BaseConfig implements VideoCodecSWConfig {
     return target;
   }
 
-  shouldScale(videoStream: VideoStreamInfo) {
+  shouldScale(videoStream: VideoStreamInfo, currentDimensions?: { width: number; height: number }) {
-    const oddDimensions = videoStream.height % 2 !== 0 || videoStream.width % 2 !== 0;
+    const dims = currentDimensions || { width: videoStream.width, height: videoStream.height };
-    const largerThanTarget = Math.min(videoStream.height, videoStream.width) > this.getTargetResolution(videoStream);
+    const oddDimensions = dims.height % 2 !== 0 || dims.width % 2 !== 0;
+    const largerThanTarget = Math.min(dims.height, dims.width) > this.getTargetResolution(videoStream);
     return oddDimensions || largerThanTarget;
   }
 
@@ -248,9 +295,11 @@ export class BaseConfig implements VideoCodecSWConfig {
     return videoStream.isHDR && this.config.tonemap !== ToneMapping.Disabled;
   }
 
-  getScaling(videoStream: VideoStreamInfo, mult = 2) {
+  getScaling(videoStream: VideoStreamInfo, mult = 2, currentDimensions?: { width: number; height: number }) {
+    const dims = currentDimensions || { width: videoStream.width, height: videoStream.height };
     const targetResolution = this.getTargetResolution(videoStream);
-    return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
+    const isVertical = dims.height > dims.width || this.isVideoRotated(videoStream);
+    return isVertical ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
   }
 
   getSize(videoStream: VideoStreamInfo) {
@@ -329,6 +378,31 @@ export class BaseConfig implements VideoCodecSWConfig {
   useCQP() {
     return this.config.cqMode === CQMode.Cqp;
   }
+
+  // Edit operations (software filters)
+  getCropOperation({ x, y, width, height }: CropParameters): string {
+    return `crop=${width}:${height}:${x}:${y}`;
+  }
+
+  getRotateOperation({ angle }: RotateParameters): string {
+    switch (angle) {
+      case 90: {
+        return 'transpose=1'; // 90° clockwise
+      }
+      case 180: {
+        return 'hflip,vflip'; // 180°
+      }
+      case 270: {
+        return 'transpose=2'; // 90° counter-clockwise (270° clockwise)
+      }
+    }
+
+    return '';
+  }
+
+  getMirrorOperation({ axis }: MirrorParameters): string {
+    return axis === MirrorAxis.Horizontal ? 'hflip' : 'vflip';
+  }
 }
 
 export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
@@ -423,14 +497,14 @@ export class ThumbnailConfig extends BaseConfig {
     return ['-fps_mode vfr', '-frames:v 1', '-update 1'];
   }
 
-  getFilterOptions(videoStream: VideoStreamInfo): string[] {
+  getFilterOptions(videoStream: VideoStreamInfo, edits: AssetEditActionItem[] = []): string[] {
     return [
       'fps=12:start_time=0:eof_action=pass:round=down',
       'thumbnail=12',
       String.raw`select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20)`,
       'trim=end_frame=2',
       'reverse',
-      ...super.getFilterOptions(videoStream),
+      ...super.getFilterOptions(videoStream, edits),
     ];
   }
 

web/src/lib/services/asset.service.ts

@@ -243,7 +243,6 @@ export const getAssetActions = ($t: MessageFormatter, asset: AssetResponseDto) =
       !sharedLink &&
       isOwner &&
       asset.type === AssetTypeEnum.Image &&
-      !asset.livePhotoVideoId &&
       asset.exifInfo?.projectionType !== ProjectionType.EQUIRECTANGULAR &&
       !asset.originalPath.toLowerCase().endsWith('.insp') &&
       !asset.originalPath.toLowerCase().endsWith('.gif') &&