always use hw filters when hwa is enabled

server/src/services/media.service.spec.ts

@@ -2133,7 +2133,7 @@ describe(MediaService.name, () => {
             '-map 0:1',
             '-g 256',
             '-v verbose',
-            '-vf scale_rkrga=-2:720:format=nv12:afbc=1',
+            '-vf scale_rkrga=-2:720:format=nv12:afbc=1:async_depth=4',
             '-level 51',
             '-rc_mode CQP',
             '-qp_init 23',
@@ -2204,7 +2204,7 @@ describe(MediaService.name, () => {
           inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']),
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
+              'scale_rkrga=-2:720:format=p010:afbc=1:async_depth=4,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
             ),
           ]),
           twoPass: false,
@@ -2212,6 +2212,28 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should set hardware decoding options for rkmpp when hardware decoding is enabled with no OpenCL on non-HDR file', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ isFile: () => false, isCharacterDevice: () => false } as Stats);
+      mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: true, crf: 30, maxBitrate: '0' },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining('scale_rkrga=-2:720:format=nv12:afbc=1:async_depth=4'),
+          ]),
+          twoPass: false,
+        }),
+      );
+    });
+
     it('should use software decoding and tone-mapping if hardware decoding is disabled', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       storageMock.stat.mockResolvedValue({ isFile: () => true, isCharacterDevice: () => true } as Stats);
@@ -2236,7 +2258,7 @@ describe(MediaService.name, () => {
       );
     });
 
-    it('should use software decoding and tone-mapping if opencl is not available', async () => {
+    it('should use software tone-mapping if opencl is not available', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       storageMock.stat.mockResolvedValue({ isFile: () => false, isCharacterDevice: () => false } as Stats);
       mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
@@ -2249,7 +2271,7 @@ describe(MediaService.name, () => {
         '/original/path.ext',
         'upload/encoded-video/user-id/as/se/asset-id.mp4',
         expect.objectContaining({
-          inputOptions: [],
+          inputOptions: expect.any(Array),
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
               'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',

server/src/services/media.service.ts

@@ -329,9 +329,11 @@ export class MediaService extends BaseService {
     }
 
     if (ffmpeg.accel === TranscodeHWAccel.DISABLED) {
-      this.logger.log(`Encoding video ${asset.id} without hardware acceleration`);
+      this.logger.log(`Transcoding video ${asset.id} without hardware acceleration`);
     } else {
-      this.logger.log(`Encoding video ${asset.id} with ${ffmpeg.accel.toUpperCase()} acceleration`);
+      this.logger.log(
+        `Transcoding video ${asset.id} with ${ffmpeg.accel.toUpperCase()}-accelerated encoding and${ffmpeg.accelDecode ? '' : ' software'} decoding`,
+      );
     }
 
     try {
@@ -341,11 +343,27 @@ export class MediaService extends BaseService {
       if (ffmpeg.accel === TranscodeHWAccel.DISABLED) {
         return JobStatus.FAILED;
       }
+
+      let partialFallbackSuccess = false;
+      if (ffmpeg.accelDecode) {
+        try {
+          this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()}-accelerated encoding and software decoding`);
+          const config = BaseConfig.create({ ...ffmpeg, accelDecode: false });
+          command = config.getCommand(target, mainVideoStream, mainAudioStream);
+          await this.mediaRepository.transcode(input, output, command);
+          partialFallbackSuccess = true;
+        } catch (error: any) {
+          this.logger.error(`Error occurred during transcoding: ${error.message}`);
+        }
+      }
+
+      if (!partialFallbackSuccess) {
         this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled`);
         const config = BaseConfig.create({ ...ffmpeg, accel: TranscodeHWAccel.DISABLED });
         command = config.getCommand(target, mainVideoStream, mainAudioStream);
         await this.mediaRepository.transcode(input, output, command);
       }
+    }
 
     this.logger.log(`Successfully encoded ${asset.id}`);
 
@@ -502,7 +520,7 @@ export class MediaService extends BaseService {
         const maliDeviceStat = await this.storageRepository.stat('/dev/mali0');
         this.maliOpenCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
       } catch {
-        this.logger.debug('OpenCL not available for transcoding, so RKMPP acceleration will use CPU decoding');
+        this.logger.debug('OpenCL not available for transcoding, so RKMPP acceleration will use CPU tonemapping');
         this.maliOpenCL = false;
       }
     }

server/src/utils/media.ts

@@ -58,9 +58,8 @@ export class BaseConfig implements VideoCodecSWConfig {
         break;
       }
       case TranscodeHWAccel.RKMPP: {
-        handler =
-          config.accelDecode && hasMaliOpenCL
-            ? new RkmppHwDecodeConfig(config, devices)
+        handler = config.accelDecode
+          ? new RkmppHwDecodeConfig(config, devices, hasMaliOpenCL)
           : new RkmppSwDecodeConfig(config, devices);
         break;
       }
@@ -327,6 +326,32 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
     this.devices = this.validateDevices(devices);
   }
 
+  getScalingFilter(videoStream: VideoStreamInfo, format: string) {
+    return `scale_${this.config.accel}=${this.getScaling(videoStream)}:format=${format}`;
+  }
+
+  getHwDecodeFilterOptions(videoStream: VideoStreamInfo) {
+    const options = [];
+    const tonemapOptions = this.getToneMapping(videoStream);
+    if (this.shouldScale(videoStream) || tonemapOptions.length === 0) {
+      const format = tonemapOptions.length === 0 ? 'nv12' : 'p010';
+      options.push(this.getScalingFilter(videoStream, format));
+    } else if (tonemapOptions.length > 0) {
+      options.push('format=p010');
+    }
+    options.push(...tonemapOptions);
+    if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) {
+      options.push('format=nv12');
+    }
+    return options;
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    return this.config.accelDecode
+      ? this.getHwDecodeFilterOptions(videoStream)
+      : [`hwupload_${this.config.accel}`, ...this.getHwDecodeFilterOptions(videoStream)];
+  }
+
   getSupportedCodecs() {
     return [VideoCodec.H264, VideoCodec.HEVC];
   }
@@ -370,6 +395,20 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
 
     return `/dev/dri/${deviceName}`;
   }
+
+  getInputThreadOptions() {
+    if (this.config.accelDecode) {
+      return [`-threads 1`];
+    }
+    return [];
+  }
+
+  getOutputThreadOptions() {
+    if (this.config.accelDecode) {
+      return [];
+    }
+    return super.getOutputThreadOptions();
+  }
 }
 
 export class ThumbnailConfig extends BaseConfig {
@@ -504,15 +543,11 @@ export class AV1Config extends BaseConfig {
   }
 }
 
-export class NvencSwDecodeConfig extends BaseHWConfig {
+export class NvencConfig extends BaseHWConfig {
   getSupportedCodecs() {
     return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.AV1];
   }
 
-  getBaseInputOptions() {
-    return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
-  }
-
   getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
     const options = [
       // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
@@ -531,16 +566,6 @@ export class NvencSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.getToneMapping(videoStream);
-    options.push('hwupload_cuda');
-    if (this.shouldScale(videoStream)) {
-      options.push(`scale_cuda=${this.getScaling(videoStream)}:format=nv12`);
-    }
-
-    return options;
-  }
-
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {
@@ -570,10 +595,6 @@ export class NvencSwDecodeConfig extends BaseHWConfig {
     }
   }
 
-  getThreadOptions() {
-    return [];
-  }
-
   getRefs() {
     const bframes = this.getBFrames();
     if (bframes > 0 && bframes < 3 && this.config.refs < 3) {
@@ -581,27 +602,14 @@ export class NvencSwDecodeConfig extends BaseHWConfig {
     }
     return this.config.refs;
   }
-}
 
-export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
   getBaseInputOptions() {
+    if (!this.config.accelDecode) {
+      return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
+    }
     return ['-hwaccel cuda', '-hwaccel_output_format cuda', '-noautorotate', ...this.getInputThreadOptions()];
   }
 
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = [];
-    if (this.shouldScale(videoStream)) {
-      options.push(`scale_cuda=${this.getScaling(videoStream)}`);
-    }
-    options.push(...this.getToneMapping(videoStream));
-    if (options.length > 0) {
-      options[options.length - 1] += ':format=nv12';
-    } else if (!videoStream.pixelFormat.endsWith('420p')) {
-      options.push('format=nv12');
-    }
-    return options;
-  }
-
   getToneMapping(videoStream: VideoStreamInfo) {
     if (!this.shouldToneMap(videoStream)) {
       return [];
@@ -617,28 +625,36 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
       'tonemap_mode=lum',
       `transfer=${transfer}`,
       'peak=100',
+      'format=nv12',
     ];
 
     return [`tonemap_cuda=${tonemapOptions.join(':')}`];
   }
-
-  getInputThreadOptions() {
-    return [`-threads 1`];
-  }
-
-  getOutputThreadOptions() {
-    return [];
-  }
 }
 
-export class QsvSwDecodeConfig extends BaseHWConfig {
+export class QsvConfig extends BaseHWConfig {
   getBaseInputOptions() {
     if (this.devices.length === 0) {
       throw new Error('No QSV device found');
     }
 
-    let qsvString = '';
     const hwDevice = this.getPreferredHardwareDevice();
+    if (this.config.accelDecode) {
+      const options = [
+        '-hwaccel qsv',
+        '-hwaccel_output_format qsv',
+        '-async_depth 4',
+        '-noautorotate',
+        ...this.getInputThreadOptions(),
+      ];
+      if (hwDevice) {
+        options.push(`-qsv_device ${hwDevice}`);
+      }
+
+      return options;
+    }
+
+    let qsvString = '';
     if (hwDevice) {
       qsvString = `,child_device=${hwDevice}`;
     }
@@ -655,15 +671,6 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.getToneMapping(videoStream);
-    options.push('hwupload=extra_hw_frames=64');
-    if (this.shouldScale(videoStream)) {
-      options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq:format=nv12`);
-    }
-    return options;
-  }
-
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {
@@ -709,44 +716,9 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
   getScaling(videoStream: VideoStreamInfo): string {
     return super.getScaling(videoStream, 1);
   }
-}
 
-export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
-  getBaseInputOptions() {
-    if (this.devices.length === 0) {
-      throw new Error('No QSV device found');
-    }
-
-    const options = [
-      '-hwaccel qsv',
-      '-hwaccel_output_format qsv',
-      '-async_depth 4',
-      '-noautorotate',
-      ...this.getInputThreadOptions(),
-    ];
-    const hwDevice = this.getPreferredHardwareDevice();
-    if (hwDevice) {
-      options.push(`-qsv_device ${hwDevice}`);
-    }
-
-    return options;
-  }
-
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = [];
-    const tonemapOptions = this.getToneMapping(videoStream);
-    if (this.shouldScale(videoStream) || tonemapOptions.length === 0) {
-      let scaling = `scale_qsv=${this.getScaling(videoStream)}:async_depth=4:mode=hq`;
-      if (tonemapOptions.length === 0) {
-        scaling += ':format=nv12';
-      }
-      options.push(scaling);
-    }
-    options.push(...tonemapOptions);
-    if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) {
-      options.push('format=nv12');
-    }
-    return options;
+  getScalingFilter(videoStream: VideoStreamInfo, format: string) {
+    return `scale_qsv=${this.getScaling(videoStream)}:async_depth=4:mode=hq:format=${format}`;
   }
 
   getToneMapping(videoStream: VideoStreamInfo): string[] {
@@ -773,19 +745,27 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
       'hwmap=derive_device=qsv:reverse=1,format=qsv',
     ];
   }
-
-  getInputThreadOptions() {
-    return [`-threads 1`];
-  }
 }
 
-export class VaapiSwDecodeConfig extends BaseHWConfig {
+export class VaapiConfig extends BaseHWConfig {
   getBaseInputOptions() {
     if (this.devices.length === 0) {
       throw new Error('No VAAPI device found');
     }
 
     let hwDevice = this.getPreferredHardwareDevice();
+    if (this.config.accelDecode) {
+      const options = [
+        '-hwaccel vaapi',
+        '-hwaccel_output_format vaapi',
+        '-noautorotate',
+        ...this.getInputThreadOptions(),
+      ];
+      if (hwDevice) {
+        options.push(`-hwaccel_device ${hwDevice}`);
+      }
+    }
+
     if (!hwDevice) {
       hwDevice = `/dev/dri/${this.devices[0]}`;
     }
@@ -793,16 +773,6 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
     return [`-init_hw_device vaapi=accel:${hwDevice}`, '-filter_hw_device accel'];
   }
 
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.getToneMapping(videoStream);
-    options.push('hwupload=extra_hw_frames=64');
-    if (this.shouldScale(videoStream)) {
-      options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc:format=nv12`);
-    }
-
-    return options;
-  }
-
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {
@@ -844,43 +814,9 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
   useCQP() {
     return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
   }
-}
 
-export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
-  getBaseInputOptions() {
-    if (this.devices.length === 0) {
-      throw new Error('No VAAPI device found');
-    }
-
-    const options = [
-      '-hwaccel vaapi',
-      '-hwaccel_output_format vaapi',
-      '-noautorotate',
-      ...this.getInputThreadOptions(),
-    ];
-    const hwDevice = this.getPreferredHardwareDevice();
-    if (hwDevice) {
-      options.push(`-hwaccel_device ${hwDevice}`);
-    }
-
-    return options;
-  }
-
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = [];
-    const tonemapOptions = this.getToneMapping(videoStream);
-    if (this.shouldScale(videoStream) || tonemapOptions.length === 0) {
-      let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`;
-      if (tonemapOptions.length === 0) {
-        scaling += ':format=nv12';
-      }
-      options.push(scaling);
-    }
-    options.push(...tonemapOptions);
-    if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) {
-      options.push('format=nv12');
-    }
-    return options;
+  getScalingFilter(videoStream: VideoStreamInfo, format: string) {
+    return `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc:format=${format}`;
   }
 
   getToneMapping(videoStream: VideoStreamInfo): string[] {
@@ -907,29 +843,33 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
       'hwmap=derive_device=vaapi:reverse=1,format=vaapi',
     ];
   }
-
-  getInputThreadOptions() {
-    return [`-threads 1`];
-  }
 }
 
-export class RkmppSwDecodeConfig extends BaseHWConfig {
+export class RkmppConfig extends BaseHWConfig {
+  protected hasMaliOpenCL: boolean;
   constructor(
     protected config: SystemConfigFFmpegDto,
     devices: string[] = [],
+    hasMaliOpenCL = false,
   ) {
     super(config, devices);
+    this.hasMaliOpenCL = hasMaliOpenCL;
   }
 
   eligibleForTwoPass(): boolean {
     return false;
   }
 
-  getBaseInputOptions(): string[] {
+  getBaseInputOptions() {
     if (this.devices.length === 0) {
       throw new Error('No RKMPP device found');
     }
-    return [];
+
+    if (this.config.accelDecode) {
+      return ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga', '-noautorotate'];
+    }
+
+    return ['-init_hw_device rkmpp=hw', '-filter_hw_device hw'];
   }
 
   getPresetOptions() {
@@ -962,32 +902,29 @@ export class RkmppSwDecodeConfig extends BaseHWConfig {
     return [VideoCodec.H264, VideoCodec.HEVC];
   }
 
-  getVideoCodec(): string {
-    return `${this.config.targetVideoCodec}_rkmpp`;
-  }
-}
-
-export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
-  getBaseInputOptions() {
-    if (this.devices.length === 0) {
-      throw new Error('No RKMPP device found');
-    }
-
-    return ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga', '-noautorotate'];
-  }
-
   getFilterOptions(videoStream: VideoStreamInfo) {
     if (this.shouldToneMap(videoStream)) {
       const { primaries, transfer, matrix } = this.getColors();
+      if (this.hasMaliOpenCL) {
         return [
-        `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
+          // use RKMPP for scaling, OpenCL for tone mapping
+          `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1:async_depth=4`,
           'hwmap=derive_device=opencl:mode=read',
           `tonemap_opencl=format=nv12:r=pc:p=${primaries}:t=${transfer}:m=${matrix}:tonemap=${this.config.tonemap}:desat=0:tonemap_mode=lum:peak=100`,
           'hwmap=derive_device=rkmpp:mode=write:reverse=1',
           'format=drm_prime',
         ];
+      }
+      return [
+        // use RKMPP for scaling, CPU for tone mapping (only works on RK3588, which supports 10-bit output)
+        `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1:async_depth=4`,
+        'hwdownload',
+        'format=p010',
+        `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=yuv420p`,
+        'hwupload',
+      ];
     } else if (this.shouldScale(videoStream)) {
-      return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`];
+      return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1:async_depth=4`];
     }
     return [];
   }