gitea-mirror/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-08-03 09:21:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): track video metadata (#28023)

Browse Source 
* track video metadata

* earlier duration check

* revert colorspace change

* duplicate constant

* formatting

* linting

* add comments

* redundant variable

* simplify tests

* use totalDuration instead of format.duration

* medium tests

* install ffmpeg

* install noble

* update test-assets commit

* make timeBase non-nullable

* linting

* use proper smallint

* add ffmpeg to mise

* simplify duration

* regenerate migration
This commit is contained in:
Mert
2026-05-01 17:03:49 +00:00
committed by GitHub
parent c0898b96ca
commit f1d8ab8aae
31 changed files with 1697 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/services/asset-media.service.ts
+4 -4
View File
@@ -351,10 +351,10 @@ export class AssetMediaService extends BaseService {
await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt));
}
await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
await this.assetRepository.upsertExif(
{ assetId: asset.id, fileSizeInByte: file.size },
{ lockedPropertiesBehavior: 'override' },
);
await this.assetRepository.upsertExif({
exif: { assetId: asset.id, fileSizeInByte: file.size },
lockedPropertiesBehavior: 'override',
});
await this.eventRepository.emit('AssetCreate', { asset });
server/src/services/asset.service.spec.ts
+12 -8
View File
@@ -187,8 +187,10 @@ describe(AssetService.name, () => {
await sut.update(authStub.admin, asset.id, { description: 'Test description' });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: asset.id, description: 'Test description', lockedProperties: ['description'] },
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: { assetId: asset.id, description: 'Test description', lockedProperties: ['description'] },
lockedPropertiesBehavior: 'append',
}),
);
});
@@ -201,12 +203,14 @@ describe(AssetService.name, () => {
await sut.update(authStub.admin, asset.id, { rating: 3 });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{
assetId: asset.id,
rating: 3,
lockedProperties: ['rating'],
},
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: {
assetId: asset.id,
rating: 3,
lockedProperties: ['rating'],
},
lockedPropertiesBehavior: 'append',
}),
);
});
server/src/services/asset.service.ts
+4 -4
View File
@@ -517,13 +517,13 @@ export class AssetService extends BaseService {
);
if (Object.keys(writes).length > 0) {
await this.assetRepository.upsertExif(
updateLockedColumns({
await this.assetRepository.upsertExif({
exif: updateLockedColumns({
assetId: id,
...writes,
}),
{ lockedPropertiesBehavior: 'append' },
);
lockedPropertiesBehavior: 'append',
});
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id } });
}
}
server/src/services/media.service.spec.ts
+154 -142
View File
File diff suppressed because it is too large Load Diff
server/src/services/media.service.ts
+13 -33
View File
@@ -15,7 +15,6 @@ import {
ImageFormat,
JobName,
JobStatus,
LogLevel,
QueueName,
RawExtractedFormat,
StorageFolder,
@@ -506,10 +505,7 @@ export class MediaService extends BaseService {
};
}
private async generateVideoThumbnails(
asset: ThumbnailPathEntity & { originalPath: string },
{ ffmpeg, image }: SystemConfig,
) {
private async generateVideoThumbnails(asset: ThumbnailAsset, { ffmpeg, image }: SystemConfig) {
const previewFile = this.getImageFile(asset, {
fileType: AssetFileType.Preview,
format: image.preview.format,
@@ -526,22 +522,15 @@ export class MediaService extends BaseService {
});
this.storageCore.ensureFolders(previewFile.path);
const { format, audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) {
throw new Error(`No video streams found for asset ${asset.id}`);
const { videoStream, format } = asset;
if (!videoStream || !format) {
throw new Error(`Missing video metadata for asset ${asset.id}`);
}
const mainAudioStream = this.getMainStream(audioStreams);
const previewConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.preview.size.toString() });
const thumbnailConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() });
const previewOptions = previewConfig.getCommand(TranscodeTarget.Video, mainVideoStream, mainAudioStream, format);
const thumbnailOptions = thumbnailConfig.getCommand(
TranscodeTarget.Video,
mainVideoStream,
mainAudioStream,
format,
);
const thumbConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() });
const previewOptions = previewConfig.getCommand(TranscodeTarget.Video, videoStream, undefined, format ?? undefined);
const thumbnailOptions = thumbConfig.getCommand(TranscodeTarget.Video, videoStream, undefined, format ?? undefined);
await this.mediaRepository.transcode(asset.originalPath, previewFile.path, previewOptions);
await this.mediaRepository.transcode(asset.originalPath, thumbnailFile.path, thumbnailOptions);
@@ -554,7 +543,7 @@ export class MediaService extends BaseService {
return {
files: [previewFile, thumbnailFile],
thumbhash,
fullsizeDimensions: { width: mainVideoStream.width, height: mainVideoStream.height },
fullsizeDimensions: { width: videoStream.width, height: videoStream.height },
};
}
@@ -588,17 +577,14 @@ export class MediaService extends BaseService {
const output = StorageCore.getEncodedVideoPath(asset);
this.storageCore.ensureFolders(output);
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, {
countFrames: this.logger.isLevelEnabled(LogLevel.Debug), // makes frame count more reliable for progress logs
});
const videoStream = this.getMainStream(videoStreams);
const audioStream = this.getMainStream(audioStreams);
if (!videoStream || !format.formatName) {
const { videoStream, format } = asset;
const audioStream = asset.audioStream ?? undefined;
if (!videoStream || !format) {
this.logger.warn(`Skipped transcoding for asset ${asset.id}: missing metadata; re-run extraction first`);
return JobStatus.Failed;
}
if (!videoStream.height || !videoStream.width) {
this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video streams found`);
this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video dimensions`);
return JobStatus.Failed;
}
@@ -667,12 +653,6 @@ export class MediaService extends BaseService {
return JobStatus.Success;
}
private getMainStream<T extends VideoStreamInfo | AudioStreamInfo>(streams: T[]): T {
return streams
.filter((stream) => stream.codecName !== 'unknown')
.toSorted((stream1, stream2) => stream2.bitrate - stream1.bitrate)[0];
}
private getTranscodeTarget(
config: SystemConfigFFmpegDto,
videoStream: VideoStreamInfo,
server/src/services/metadata.service.spec.ts
+271 -90
View File
@@ -18,7 +18,7 @@ import { ImmichTags } from 'src/repositories/metadata.repository';
import { firstDateTime, MetadataService } from 'src/services/metadata.service';
import { AssetFactory } from 'test/factories/asset.factory';
import { PersonFactory } from 'test/factories/person.factory';
import { probeStub } from 'test/fixtures/media.stub';
import { videoInfoStub } from 'test/fixtures/media.stub';
import { tagStub } from 'test/fixtures/tag.stub';
import { getForMetadataExtraction, getForSidecarWrite } from 'test/mappers';
import { factory } from 'test/small.factory';
@@ -59,6 +59,15 @@ const makeFaceTags = (face: Partial<{ Name: string }> = {}, orientation?: Immich
},
});
const emptyPackets = {
totalDuration: 0,
packetCount: 0,
outputFrames: 0,
keyframePts: [],
keyframeAccDuration: [],
keyframeOwnDuration: [],
};
describe(MetadataService.name, () => {
let sut: MetadataService;
let mocks: ServiceMocks;
@@ -183,9 +192,12 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }), {
lockedPropertiesBehavior: 'skip',
});
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
exif: expect.objectContaining({ dateTimeOriginal: sidecarDate }),
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
id: asset.id,
@@ -212,8 +224,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -242,8 +256,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -265,9 +281,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),
exif: expect.objectContaining({
dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith(
@@ -290,9 +308,12 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }), {
lockedPropertiesBehavior: 'skip',
});
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
exif: expect.objectContaining({ iso: 160 }),
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
duration: null,
@@ -323,8 +344,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ city: null, state: null, country: null }),
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: expect.objectContaining({ city: null, state: null, country: null }),
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -353,8 +376,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -378,8 +403,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ latitude: null, longitude: null }),
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: expect.objectContaining({ latitude: null, longitude: null }),
lockedPropertiesBehavior: 'skip',
}),
);
});
@@ -585,7 +612,7 @@ describe(MetadataService.name, () => {
it('should not apply motion photos if asset is video', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
mocks.media.probe.mockResolvedValue(videoInfoStub.matroskaContainer);
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
@@ -611,15 +638,144 @@ describe(MetadataService.name, () => {
it('should extract the correct video orientation', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamVertical2160p);
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),
lockedPropertiesBehavior: 'skip',
}),
);
});
it('should persist CICP smallints and profile/level for HDR10 video', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
mocks.media.probePackets.mockResolvedValue(emptyPackets);
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
exif: expect.objectContaining({ fps: 59.94 }),
video: expect.objectContaining({
codecName: 'hevc',
profile: 2,
level: 153,
pixelFormat: 'yuv420p10le',
colorPrimaries: 9,
colorTransfer: 16,
colorMatrix: 9,
dvProfile: undefined,
}),
}),
);
});
it('should persist Dolby Vision fields', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamDolbyVision);
mocks.media.probePackets.mockResolvedValue(emptyPackets);
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
video: expect.objectContaining({
dvProfile: 8,
dvLevel: 10,
dvBlSignalCompatibilityId: 4,
colorTransfer: 18, // ARIB_STD_B67
}),
}),
);
});
it('should persist packet-derived HLS fields', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
mocks.media.probePackets.mockResolvedValue({
totalDuration: 12_080,
packetCount: 1148,
outputFrames: 1149,
keyframePts: [-590, 10, 611, 1211],
keyframeAccDuration: [10, 610, 6110, 12_080],
keyframeOwnDuration: [10, 10, 10, 10],
});
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
video: expect.objectContaining({ timeBase: 600 }),
keyframes: expect.objectContaining({
totalDuration: 12_080,
packetCount: 1148,
outputFrames: 1149,
pts: [-590, 10, 611, 1211],
accDuration: [10, 610, 6110, 12_080],
ownDuration: [10, 10, 10, 10],
}),
}),
);
});
it('should omit the keyframe row when the probe returns no keyframes', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
mocks.media.probePackets.mockResolvedValue(emptyPackets);
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.not.objectContaining({ keyframes: expect.anything() }),
);
});
it('should prefer ffprobe frameRate over exiftool VideoFrameRate', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
mocks.media.probePackets.mockResolvedValue(emptyPackets);
mockReadTags({ VideoFrameRate: '30' });
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
exif: expect.objectContaining({ fps: 59.94 }),
lockedPropertiesBehavior: 'skip',
}),
);
});
it('should not insert audio/video/keyframe rows for image assets', async () => {
const asset = AssetFactory.create({ type: AssetType.Image });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.media.probe).not.toHaveBeenCalled();
expect(mocks.media.probePackets).not.toHaveBeenCalled();
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.not.objectContaining({
audio: expect.anything(),
video: expect.anything(),
keyframes: expect.anything(),
}),
);
});
@@ -909,39 +1065,41 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{
assetId: asset.id,
bitsPerSample: expect.any(Number),
autoStackId: null,
colorspace: tags.ColorSpace,
dateTimeOriginal: dateForTest,
description: tags.ImageDescription,
exifImageHeight: null,
exifImageWidth: null,
exposureTime: tags.ExposureTime,
fNumber: null,
fileSizeInByte: 123_456,
focalLength: tags.FocalLength,
fps: null,
iso: tags.ISO,
latitude: null,
lensModel: tags.LensModel,
livePhotoCID: tags.MediaGroupUUID,
longitude: null,
make: tags.Make,
model: tags.Model,
modifyDate: expect.any(Date),
orientation: tags.Orientation?.toString(),
profileDescription: tags.ProfileDescription,
projectionType: 'EQUIRECTANGULAR',
timeZone: tags.zone,
rating: tags.Rating,
country: null,
state: null,
city: null,
tags: ['parent/child'],
},
{ lockedPropertiesBehavior: 'skip' },
expect.objectContaining({
exif: {
assetId: asset.id,
bitsPerSample: expect.any(Number),
autoStackId: null,
colorspace: tags.ColorSpace,
dateTimeOriginal: dateForTest,
description: tags.ImageDescription,
exifImageHeight: null,
exifImageWidth: null,
exposureTime: tags.ExposureTime,
fNumber: null,
fileSizeInByte: 123_456,
focalLength: tags.FocalLength,
fps: null,
iso: tags.ISO,
latitude: null,
lensModel: tags.LensModel,
livePhotoCID: tags.MediaGroupUUID,
longitude: null,
make: tags.Make,
model: tags.Model,
modifyDate: expect.any(Date),
orientation: tags.Orientation?.toString(),
profileDescription: tags.ProfileDescription,
projectionType: 'EQUIRECTANGULAR',
timeZone: tags.zone,
rating: tags.Rating,
country: null,
state: null,
city: null,
tags: ['parent/child'],
},
lockedPropertiesBehavior: 'skip',
}),
);
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
@@ -975,9 +1133,11 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
timeZone: 'UTC+0',
exif: expect.objectContaining({
timeZone: 'UTC+0',
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -985,9 +1145,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
...probeStub.videoStreamH264,
...videoInfoStub.videoStreamH264,
format: {
...probeStub.videoStreamH264.format,
...videoInfoStub.videoStreamH264.format,
duration: 6.21,
},
});
@@ -1008,9 +1168,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create();
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
...probeStub.videoStreamH264,
...videoInfoStub.videoStreamH264,
format: {
...probeStub.videoStreamH264.format,
...videoInfoStub.videoStreamH264.format,
duration: 6.21,
},
});
@@ -1030,9 +1190,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
...probeStub.videoStreamH264,
...videoInfoStub.videoStreamH264,
format: {
...probeStub.videoStreamH264.format,
...videoInfoStub.videoStreamH264.format,
duration: 0,
},
});
@@ -1053,9 +1213,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
...probeStub.videoStreamH264,
...videoInfoStub.videoStreamH264,
format: {
...probeStub.videoStreamH264.format,
...videoInfoStub.videoStreamH264.format,
duration: 604_800,
},
});
@@ -1111,9 +1271,9 @@ describe(MetadataService.name, () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mockReadTags({ Duration: 123 }, {});
mocks.media.probe.mockResolvedValue({
...probeStub.videoStreamH264,
...videoInfoStub.videoStreamH264,
format: {
...probeStub.videoStreamH264.format,
...videoInfoStub.videoStreamH264.format,
duration: 456,
},
});
@@ -1132,18 +1292,22 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
description: '',
exif: expect.objectContaining({
description: '',
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
mockReadTags({ ImageDescription: ' my\n description' });
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
description: 'my\n description',
exif: expect.objectContaining({
description: 'my\n description',
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1155,9 +1319,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
description: '1000',
exif: expect.objectContaining({
description: '1000',
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1388,9 +1554,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
modifyDate: expect.any(Date),
exif: expect.objectContaining({
modifyDate: expect.any(Date),
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1402,9 +1570,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
rating: null,
exif: expect.objectContaining({
rating: null,
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1416,9 +1586,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
rating: 5,
exif: expect.objectContaining({
rating: 5,
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1430,9 +1602,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
rating: null,
exif: expect.objectContaining({
rating: null,
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1444,9 +1618,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
rating: -1,
exif: expect.objectContaining({
rating: -1,
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
@@ -1466,7 +1642,7 @@ describe(MetadataService.name, () => {
it('should handle not finding a match', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.media.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamVertical2160p);
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mockReadTags({ ContentIdentifier: 'CID' });
@@ -1578,9 +1754,12 @@ describe(MetadataService.name, () => {
mockReadTags(exif);
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected), {
lockedPropertiesBehavior: 'skip',
});
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
exif: expect.objectContaining(expected),
lockedPropertiesBehavior: 'skip',
}),
);
});
it.each([
@@ -1605,9 +1784,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
lensModel: expected,
exif: expect.objectContaining({
lensModel: expected,
}),
lockedPropertiesBehavior: 'skip',
}),
{ lockedPropertiesBehavior: 'skip' },
);
});
server/src/services/metadata.service.ts
+80 -17
View File
@@ -243,10 +243,11 @@ export class MetadataService extends BaseService {
return;
}
const [exifTags, stats] = await Promise.all([
const [exifResult, stats] = await Promise.all([
this.getExifTags(asset),
this.storageRepository.stat(asset.originalPath),
]);
const { tags: exifTags, audio, video, packets, format } = exifResult;
this.logger.verbose('Exif Tags', exifTags);
const dates = this.getDates(asset, exifTags, stats);
@@ -294,7 +295,7 @@ export class MetadataService extends BaseService {
exifTags.Make ?? exifTags.Device?.Manufacturer ?? exifTags.AndroidMake ?? (exifTags.DeviceManufacturer || null),
model:
exifTags.Model ?? exifTags.Device?.ModelName ?? exifTags.AndroidModel ?? (exifTags.DeviceModelName || null),
fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)),
fps: video?.frameRate ?? validate(Number.parseFloat(exifTags.VideoFrameRate!)),
iso: validate(exifTags.ISO) as number,
exposureTime: exifTags.ExposureTime ?? null,
lensModel: getLensModel(exifTags),
@@ -313,6 +314,53 @@ export class MetadataService extends BaseService {
tags: tags.length > 0 ? tags : null,
};
const audioData =
format && audio?.codecName
? {
assetId: asset.id,
bitrate: audio.bitrate,
index: audio.index,
profile: audio.profile,
codecName: audio.codecName,
}
: undefined;
const videoData =
format?.formatName && format?.formatLongName && video?.codecName && video?.timeBase
? {
assetId: asset.id,
bitrate: video.bitrate,
frameCount: video.frameCount,
timeBase: video.timeBase,
index: video.index,
profile: video.profile,
level: video.level,
colorPrimaries: video.colorPrimaries,
colorTransfer: video.colorTransfer,
colorMatrix: video.colorMatrix,
dvProfile: video.dvProfile,
dvLevel: video.dvLevel,
dvBlSignalCompatibilityId: video.dvBlSignalCompatibilityId,
codecName: video.codecName,
formatName: format.formatName,
formatLongName: format.formatLongName,
pixelFormat: video.pixelFormat,
}
: undefined;
const keyframeData =
packets && packets.keyframePts.length > 0
? {
assetId: asset.id,
totalDuration: packets.totalDuration,
packetCount: packets.packetCount,
outputFrames: packets.outputFrames,
pts: packets.keyframePts,
accDuration: packets.keyframeAccDuration,
ownDuration: packets.keyframeOwnDuration,
}
: undefined;
const isSidewards = exifTags.Orientation && this.isOrientationSidewards(exifTags.Orientation);
const assetWidth = isSidewards ? validate(height) : validate(width);
const assetHeight = isSidewards ? validate(width) : validate(height);
@@ -333,7 +381,13 @@ export class MetadataService extends BaseService {
height: !asset.isEdited || asset.height == null ? assetHeight : undefined,
}),
async () => {
await this.assetRepository.upsertExif(exifData, { lockedPropertiesBehavior: 'skip' });
await this.assetRepository.upsertExif({
exif: exifData,
audio: audioData,
video: videoData,
keyframes: keyframeData,
lockedPropertiesBehavior: 'skip',
});
await this.applyTagList(asset);
},
);
@@ -523,13 +577,14 @@ export class MetadataService extends BaseService {
return { width, height };
}
private async getExifTags(asset: { originalPath: string; files: AssetFile[]; type: AssetType }): Promise<ImmichTags> {
private async getExifTags(asset: { originalPath: string; files: AssetFile[]; type: AssetType }) {
const { sidecarFile } = getAssetFiles(asset.files);
const shouldProbe = asset.type === AssetType.Video || asset.originalPath.toLowerCase().endsWith('.gif');
const [mediaTags, sidecarTags, videoTags] = await Promise.all([
const [mediaTags, sidecarTags, videoResult] = await Promise.all([
this.metadataRepository.readTags(asset.originalPath),
sidecarFile ? this.metadataRepository.readTags(sidecarFile.path) : null,
asset.type === AssetType.Video ? this.getVideoTags(asset.originalPath) : null,
shouldProbe ? this.getVideoTags(asset.originalPath) : null,
]);
// prefer dates from sidecar tags
@@ -554,14 +609,20 @@ export class MetadataService extends BaseService {
// prefer duration from video tags
// don't save duration if asset is definitely not an animated image (see e.g. CR3 with Duration: 1s)
if (videoTags || !mimeTypes.isPossiblyAnimatedImage(asset.originalPath)) {
if (videoResult || !mimeTypes.isPossiblyAnimatedImage(asset.originalPath)) {
delete mediaTags.Duration;
}
// never use duration from sidecar
delete sidecarTags?.Duration;
return { ...mediaTags, ...videoTags, ...sidecarTags };
return {
tags: { ...mediaTags, ...videoResult?.tags, ...sidecarTags },
audio: videoResult?.audio,
video: videoResult?.video,
packets: videoResult?.packets,
format: videoResult?.format ?? null,
};
}
private getTagList(exifTags: ImmichTags): string[] {
@@ -1008,20 +1069,22 @@ export class MetadataService extends BaseService {
}
private async getVideoTags(originalPath: string) {
const { videoStreams, format } = await this.mediaRepository.probe(originalPath);
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(originalPath);
const video = videoStreams[0];
const audio = audioStreams[0];
const packets = video?.timeBase ? await this.mediaRepository.probePackets(originalPath, video.index) : null;
const tags: Pick<ImmichTags, 'Duration' | 'Orientation' | 'ImageWidth' | 'ImageHeight'> = {};
if (videoStreams[0]) {
// Set video dimensions
if (videoStreams[0].width) {
tags.ImageWidth = videoStreams[0].width;
if (video) {
if (video.width) {
tags.ImageWidth = video.width;
}
if (videoStreams[0].height) {
tags.ImageHeight = videoStreams[0].height;
if (video.height) {
tags.ImageHeight = video.height;
}
switch (videoStreams[0].rotation) {
switch (video.rotation) {
case -90: {
tags.Orientation = ExifOrientation.Rotate90CW;
break;
@@ -1045,6 +1108,6 @@ export class MetadataService extends BaseService {
tags.Duration = format.duration;
}
return tags;
return { tags, audio, video, packets, format };
}
}
server/src/services/tag.service.spec.ts
+20 -10
View File
@@ -206,16 +206,22 @@ describe(TagService.name, () => {
count: 6,
});
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: { assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
lockedPropertiesBehavior: 'append',
}),
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: { assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
lockedPropertiesBehavior: 'append',
}),
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-3', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: { assetId: 'asset-3', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
lockedPropertiesBehavior: 'append',
}),
);
expect(mocks.tag.upsertAssetIds).toHaveBeenCalledWith([
{ tagId: 'tag-1', assetId: 'asset-1' },
@@ -255,12 +261,16 @@ describe(TagService.name, () => {
]);
expect(mocks.asset.upsertExif).not.toHaveBeenCalledWith(
{ assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1'] },
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: { assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1'] },
lockedPropertiesBehavior: 'append',
}),
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
{ assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1'] },
{ lockedPropertiesBehavior: 'append' },
expect.objectContaining({
exif: { assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1'] },
lockedPropertiesBehavior: 'append',
}),
);
expect(mocks.tag.getAssetIds).toHaveBeenCalledWith('tag-1', ['asset-1', 'asset-2']);
expect(mocks.tag.addAssetIds).toHaveBeenCalledWith('tag-1', ['asset-2']);
server/src/services/tag.service.ts
+2 -1
View File
@@ -152,7 +152,8 @@ export class TagService extends BaseService {
private async updateTags(assetId: string) {
const { tags } = await this.assetRepository.getForUpdateTags(assetId);
await this.assetRepository.upsertExif(updateLockedColumns({ assetId, tags: tags.map(({ value }) => value) }), {
await this.assetRepository.upsertExif({
exif: updateLockedColumns({ assetId, tags: tags.map(({ value }) => value) }),
lockedPropertiesBehavior: 'append',
});
}