linting

server/src/queries/asset.job.repository.sql

@@ -627,9 +627,12 @@ select
           "asset_audio"."profile",
           "asset_audio"."bitrate"
         from
-          "asset_audio"
+          (
+            select
+              1
+          ) as "dummy"
         where
-          "asset_audio"."assetId" = "asset"."id"
+          "asset_audio"."assetId" is not null
       ) as obj
   ) as "audioStream",
   (
@@ -695,7 +698,8 @@ select
 from
   "asset"
   inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
-  left join "asset_video" on "asset_video"."assetId" = "asset"."id"
+  inner join "asset_video" on "asset_video"."assetId" = "asset"."id"
+  left join "asset_audio" on "asset_audio"."assetId" = "asset"."id"
 where
   "asset"."id" = $1
   and "asset"."type" = 'VIDEO'

server/src/repositories/asset-job.repository.ts

@@ -9,7 +9,7 @@ import { DB } from 'src/schema';
 import {
   anyUuid,
   asUuid,
-  withAudioVideo,
+  withAudioStream,
   withDefaultVisibility,
   withEdits,
   withExif,
@@ -17,6 +17,8 @@ import {
   withFaces,
   withFilePath,
   withFiles,
+  withVideoFormat,
+  withVideoStream,
 } from 'src/utils/database';
 import { mimeTypes } from 'src/utils/mime-types';
 
@@ -135,7 +137,9 @@ export class AssetJobRepository {
       )
       .select(withEdits)
       .$call(withExifInner)
-      .$call(withAudioVideo)
+      .leftJoin('asset_video', 'asset_video.assetId', 'asset.id')
+      .select((eb) => withVideoStream(eb).as('videoStream'))
+      .select((eb) => withVideoFormat(eb).as('format'))
       .where('asset.id', '=', id)
       .executeTakeFirst();
   }
@@ -336,9 +340,13 @@ export class AssetJobRepository {
     return this.db
       .selectFrom('asset')
       .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
+      .innerJoin('asset_video', 'asset_video.assetId', 'asset.id')
+      .leftJoin('asset_audio', 'asset_audio.assetId', 'asset.id')
       .select(['asset.id', 'asset.ownerId', 'asset.originalPath'])
       .select(withFiles)
-      .$call((qb) => withAudioVideo(qb, true))
+      .select((eb) => withAudioStream(eb).as('audioStream'))
+      .select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
+      .select((eb) => withVideoFormat(eb).$notNull().as('format'))
       .where('asset.id', '=', id)
       .where('asset.type', '=', sql.lit(AssetType.Video))
       .executeTakeFirst();

server/src/schema/migrations/1776823683957-CreateAudioVideoTables.ts

@@ -14,7 +14,7 @@ export async function up(db: Kysely<any>): Promise<void> {
   "assetId" uuid NOT NULL,
   "bitrate" integer NOT NULL,
   "frameCount" integer NOT NULL,
-  "timeBase" integer,
+  "timeBase" integer NOT NULL,
   "index" smallint NOT NULL,
   "profile" smallint,
   "level" smallint,

server/src/schema/tables/asset-av.table.ts

@@ -32,8 +32,8 @@ export class AssetVideoTable {
   @Column({ type: 'integer' })
   frameCount!: number;
 
-  @Column({ type: 'integer', nullable: true })
+  @Column({ type: 'integer' })
-  timeBase!: number | null;
+  timeBase!: number;
 
   @Column({ type: smallint })
   index!: number;

server/src/services/media.service.spec.ts

@@ -1,4 +1,4 @@
-import { ShallowDehydrateObject } from 'kysely';
+import { NotNull, ShallowDehydrateObject } from 'kysely';
 import { OutputInfo } from 'sharp';
 import { SystemConfig } from 'src/config';
 import { Exif } from 'src/database';
@@ -508,7 +508,7 @@ describe(MediaService.name, () => {
         expect.any(String),
         expect.objectContaining({
           inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'],
-          outputOptions: [
+          outputOptions: expect.arrayContaining([
             '-fps_mode',
             'vfr',
             '-frames:v',
@@ -519,7 +519,7 @@ describe(MediaService.name, () => {
             'verbose',
             '-vf',
             String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc`,
-          ],
+          ]),
           twoPass: false,
         }),
       );
@@ -557,7 +557,7 @@ describe(MediaService.name, () => {
         expect.any(String),
         expect.objectContaining({
           inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'],
-          outputOptions: [
+          outputOptions: expect.arrayContaining([
             '-fps_mode',
             'vfr',
             '-frames:v',
@@ -567,8 +567,8 @@ describe(MediaService.name, () => {
             '-v',
             'verbose',
             '-vf',
-            String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
+            String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:250:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
-          ],
+          ]),
           twoPass: false,
         }),
       );
@@ -608,7 +608,7 @@ describe(MediaService.name, () => {
         expect.any(String),
         expect.objectContaining({
           inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'],
-          outputOptions: [
+          outputOptions: expect.arrayContaining([
             '-fps_mode',
             'vfr',
             '-frames:v',
@@ -618,8 +618,8 @@ describe(MediaService.name, () => {
             '-v',
             'verbose',
             '-vf',
-            String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
+            String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:250:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
-          ],
+          ]),
           twoPass: false,
         }),
       );
@@ -1937,16 +1937,16 @@ describe(MediaService.name, () => {
 
   describe('handleVideoConversion', () => {
     let asset: ReturnType<typeof AssetFactory.create> & {
-      videoStream: VideoStreamInfo | null;
+      videoStream: VideoStreamInfo & { timeBase: NotNull };
       audioStream: AudioStreamInfo | null;
-      format: VideoFormat | null;
+      format: VideoFormat;
     };
     beforeEach(() => {
       asset = {
         ...AssetFactory.create({ id: 'video-id', type: AssetType.Video, originalPath: '/original/path.ext' }),
-        videoStream: null,
+        videoStream: probeStub.videoStreamH264.videoStream,
         audioStream: null,
-        format: null,
+        format: probeStub.videoStreamH264.format,
       };
       mocks.assetJob.getForVideoConversion.mockResolvedValue(asset);
       sut.videoInterfaces = { dri: ['renderD128'], mali: true };

server/src/services/metadata.service.ts

@@ -326,7 +326,7 @@ export class MetadataService extends BaseService {
         : undefined;
 
     const videoData =
-      format?.formatName && format?.formatLongName && video?.codecName
+      format?.formatName && format?.formatLongName && video?.codecName && video?.timeBase
         ? {
             assetId: asset.id,
             bitrate: video.bitrate,

server/src/utils/database.ts

@@ -101,83 +101,75 @@ export function withExifInner<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
 
 export const dummy = sql`(select 1)`.as('dummy');
 
-export function withAudioVideo<O>(qb: SelectQueryBuilder<DB, 'asset' | 'asset_exif', O>, withAudio = false) {
+export function withAudioStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_audio'>) {
-  return qb
+  return jsonObjectFrom(
-    .$if(withAudio, (qb) =>
+    eb
-      qb.select((eb) =>
+      .selectFrom(dummy)
-        jsonObjectFrom(
+      .select(['asset_audio.index', 'asset_audio.codecName', 'asset_audio.profile', 'asset_audio.bitrate'])
-          eb
+      .where('asset_audio.assetId', 'is not', sql.lit(null))
-            .selectFrom('asset_audio')
+      .$castTo<AudioStreamInfo | null>(),
-            .select(['asset_audio.index', 'asset_audio.codecName', 'asset_audio.profile', 'asset_audio.bitrate'])
+  );
-            .whereRef('asset_audio.assetId', '=', 'asset.id'),
+}
-        )
+
-          .$castTo<AudioStreamInfo | null>()
+export function withVideoStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_video'>) {
-          .as('audioStream'),
+  return jsonObjectFrom(
-      ),
+    eb
-    )
+      .selectFrom(dummy)
-    .leftJoin('asset_video', 'asset_video.assetId', 'asset.id')
+      .select((eb) => [
-    .select((eb) =>
+        'asset_video.index',
-      jsonObjectFrom(
+        'asset_video.codecName',
+        'asset_video.profile',
+        'asset_video.level',
+        'asset_video.bitrate',
+        'asset_exif.exifImageWidth as width',
+        'asset_exif.exifImageHeight as height',
+        'asset_video.pixelFormat',
+        'asset_video.frameCount',
+        'asset_exif.fps as frameRate',
+        'asset_video.timeBase',
         eb
-          .selectFrom(dummy)
+          .case()
-          .where('asset_video.assetId', 'is not', sql.lit(null))
+          .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate90CW.toString()))
-          .select((eb) => [
+          .then(sql.lit(-90))
-            'asset_video.index',
+          .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate270CW.toString()))
-            'asset_video.codecName',
+          .then(sql.lit(90))
-            'asset_video.profile',
+          .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate180.toString()))
-            'asset_video.level',
+          .then(sql.lit(180))
-            'asset_video.bitrate',
+          .else(0)
-            'asset_exif.exifImageWidth as width',
+          .end()
-            'asset_exif.exifImageHeight as height',
+          .as('rotation'),
-            'asset_video.pixelFormat',
+        'asset_video.colorPrimaries',
-            'asset_video.frameCount',
+        'asset_video.colorMatrix',
-            'asset_exif.fps as frameRate',
+        'asset_video.colorTransfer',
-            'asset_video.timeBase',
+        'asset_video.dvProfile',
-            eb
+        'asset_video.dvLevel',
-              .case()
+        'asset_video.dvBlSignalCompatibilityId',
-              .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate90CW.toString()))
+      ])
-              .then(sql.lit(-90))
+      .where('asset_video.assetId', 'is not', sql.lit(null)),
-              .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate270CW.toString()))
+  ).$castTo<(VideoStreamInfo & { timeBase: NotNull }) | null>();
-              .then(sql.lit(90))
+}
-              .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate180.toString()))
+
-              .then(sql.lit(180))
+export function withVideoFormat(eb: ExpressionBuilder<DB, 'asset' | 'asset_video'>) {
-              .else(0)
+  return jsonObjectFrom(
-              .end()
+    eb
-              .as('rotation'),
+      .selectFrom(dummy)
-            'asset_video.colorPrimaries',
+      .select([
-            'asset_video.colorMatrix',
+        'asset_video.formatName',
-            'asset_video.colorTransfer',
+        'asset_video.formatLongName',
-            'asset_video.dvProfile',
+        // TODO: simplify after https://github.com/immich-app/immich/pull/28003
-            'asset_video.dvLevel',
-            'asset_video.dvBlSignalCompatibilityId',
-          ])
-          .$castTo<VideoStreamInfo | null>(),
-      ).as('videoStream'),
-    )
-    .select((eb) =>
-      jsonObjectFrom(
         eb
-          .selectFrom(dummy)
+          .case()
-          .where('asset_video.assetId', 'is not', sql.lit(null))
+          .when('asset.duration', '~', sql<string>`'^\\d{2}:\\d{2}:\\d{2}\\.\\d{3}$'`)
-          .select((eb) => [
+          .then(
-            'asset_video.formatName',
+            sql<number>`substr(asset.duration, 1, 2)::int * 3600000 + substr(asset.duration, 4, 2)::int * 60000 + substr(asset.duration, 7, 2)::int * 1000 + substr(asset.duration, 10, 3)::int`,
-            'asset_video.formatLongName',
+          )
-            // TODO: simplify after https://github.com/immich-app/immich/pull/28003
+          .else(sql.lit(0))
-            eb
+          .end()
-              .case()
+          .as('duration'),
-              .when('asset.duration', '~', sql<string>`'^\\d{2}:\\d{2}:\\d{2}\\.\\d{3}$'`)
+        'asset_video.bitrate',
-              .then(
+      ])
-                sql<number>`substr(asset.duration, 1, 2)::int * 3600000 + substr(asset.duration, 4, 2)::int * 60000 + substr(asset.duration, 7, 2)::int * 1000 + substr(asset.duration, 10, 3)::int`,
+      .where('asset_video.assetId', 'is not', sql.lit(null)),
-              )
+  ).$castTo<VideoFormat | null>();
-              .else(sql.lit(0))
-              .end()
-              .as('duration'),
-            'asset_video.bitrate',
-          ]),
-      )
-        .$castTo<VideoFormat | null>()
-        .as('format'),
-    );
 }
 
 export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {

server/test/fixtures/media.stub.ts

@@ -1,3 +1,4 @@
+import { NotNull } from 'kysely';
 import { ColorMatrix, ColorPrimaries, ColorTransfer, DvProfile, DvSignalCompatibility } from 'src/enum';
 import { AudioStreamInfo, VideoFormat, VideoInfo, VideoStreamInfo } from 'src/types';
 
@@ -392,9 +393,9 @@ export const videoInfoStub = {
 };
 
 interface SelectedStreams {
-  videoStream: VideoStreamInfo | null;
+  videoStream: VideoStreamInfo & { timeBase: NotNull };
   audioStream: AudioStreamInfo | null;
-  format: VideoFormat | null;
+  format: VideoFormat;
 }
 
 const toSelectedStreams = (info: VideoInfo) => ({

server/test/mappers.ts

@@ -1,4 +1,4 @@
-import { Selectable, ShallowDehydrateObject } from 'kysely';
+import { NotNull, Selectable, ShallowDehydrateObject } from 'kysely';
 import { MapAsset } from 'src/dtos/asset-response.dto';
 import { AssetEditActionItem } from 'src/dtos/editing.dto';
 import { ActivityTable } from 'src/schema/tables/activity.table';
@@ -156,7 +156,7 @@ export const getForGenerateThumbnail = (asset: ReturnType<AssetFactory['build']>
   files: asset.files.map((file) => getDehydrated(file)),
   exifInfo: getDehydrated(asset.exifInfo),
   edits: asset.edits.map(({ action, parameters }) => ({ action, parameters })) as AssetEditActionItem[],
-  videoStream: null as VideoStreamInfo | null,
+  videoStream: null as (VideoStreamInfo & { timeBase: NotNull }) | null,
   audioStream: null as AudioStreamInfo | null,
   format: null as VideoFormat | null,
 });