From b52d5d4ddbeab5ec69e62e796306306a2af01e98 Mon Sep 17 00:00:00 2001 From: timonrieger Date: Thu, 23 Jul 2026 18:12:59 +0200 Subject: [PATCH] repos --- server/src/queries/search.repository.sql | 320 +++++++++++++++++++ server/src/repositories/search.repository.ts | 51 ++- server/src/utils/database.ts | 31 ++ 3 files changed, 396 insertions(+), 6 deletions(-) diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index efd7236bb5..3278afb97c 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -455,6 +455,8 @@ order by "asset"."id" desc limit $2 +offset + $3 -- SearchRepository.searchMetadataV3 (empty) select @@ -496,6 +498,8 @@ order by "asset"."id" desc limit $1 +offset + $2 -- SearchRepository.searchMetadataV3 (or-exif-only) select @@ -538,6 +542,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (string-eq-null) select @@ -580,6 +586,8 @@ order by "asset"."id" desc limit $2 +offset + $3 -- SearchRepository.searchMetadataV3 (string-pattern-like) select @@ -622,6 +630,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (string-pattern-notLike) select @@ -664,6 +674,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (string-pattern-startsWith) select @@ -706,6 +718,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (string-similarity-ocr) select @@ -755,6 +769,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (ids-any) select @@ -804,6 +820,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (ids-all) select @@ -860,6 +878,8 @@ order by "asset"."id" desc limit $5 +offset + $6 -- SearchRepository.searchMetadataV3 (ids-all-single) select @@ -909,6 +929,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (ids-none) select @@ -959,6 +981,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (ids-tags-all) select @@ -1014,6 +1038,8 @@ order by "asset"."id" desc limit $4 +offset + $5 -- SearchRepository.searchMetadataV3 (has-albums-false) select @@ -1062,6 +1088,8 @@ order by "asset"."id" desc limit $2 +offset + $3 -- SearchRepository.searchMetadataV3 (is-encoded) select @@ -1111,6 +1139,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (number-range) select @@ -1156,6 +1186,8 @@ order by "asset"."id" desc limit $4 +offset + $5 -- SearchRepository.searchMetadataV3 (date-eq) select @@ -1198,6 +1230,8 @@ order by "asset"."id" desc limit $3 +offset + $4 -- SearchRepository.searchMetadataV3 (date-range) select @@ -1243,6 +1277,8 @@ order by "asset"."id" desc limit $4 +offset + $5 -- SearchRepository.searchMetadataV3 (order-fileSize-noExif) select @@ -1285,6 +1321,8 @@ order by "asset"."id" desc limit $2 +offset + $3 -- SearchRepository.searchMetadataV3 (order-rating-withExif) select @@ -1328,6 +1366,8 @@ order by "asset"."id" asc limit $2 +offset + $3 -- SearchRepository.searchMetadataV3 (or-branches) select @@ -1382,6 +1422,8 @@ order by "asset"."id" desc limit $5 +offset + $6 -- SearchRepository.searchMetadataV3 (or-with-top-level) select @@ -1438,6 +1480,284 @@ order by "asset"."id" desc limit $6 +offset + $7 + +-- SearchRepository.searchMetadataV3 (cursor-offset) +select + "asset"."id", + "asset"."updateId", + "asset"."createdAt", + "asset"."updatedAt", + "asset"."deletedAt", + "asset"."status", + "asset"."checksum", + "asset"."checksumAlgorithm", + "asset"."duplicateId", + "asset"."duration", + "asset"."fileCreatedAt", + "asset"."fileModifiedAt", + "asset"."isExternal", + "asset"."isFavorite", + "asset"."isOffline", + "asset"."isEdited", + "asset"."visibility", + "asset"."libraryId", + "asset"."livePhotoVideoId", + "asset"."localDateTime", + "asset"."originalFileName", + "asset"."originalPath", + "asset"."ownerId", + "asset"."stackId", + "asset"."thumbhash", + "asset"."type", + "asset"."width", + "asset"."height" +from + "asset" + left join "asset_exif" on "asset"."id" = "asset_exif"."assetId" +where + "asset"."ownerId" = any ($1::uuid[]) + and "asset"."isFavorite" = $2 +order by + "asset"."fileCreatedAt" desc, + "asset"."id" desc +limit + $3 +offset + $4 + +-- SearchRepository.searchRandomV3 (baseline) +select + "asset"."id", + "asset"."updateId", + "asset"."createdAt", + "asset"."updatedAt", + "asset"."deletedAt", + "asset"."status", + "asset"."checksum", + "asset"."checksumAlgorithm", + "asset"."duplicateId", + "asset"."duration", + "asset"."fileCreatedAt", + "asset"."fileModifiedAt", + "asset"."isExternal", + "asset"."isFavorite", + "asset"."isOffline", + "asset"."isEdited", + "asset"."visibility", + "asset"."libraryId", + "asset"."livePhotoVideoId", + "asset"."localDateTime", + "asset"."originalFileName", + "asset"."originalPath", + "asset"."ownerId", + "asset"."stackId", + "asset"."thumbhash", + "asset"."type", + "asset"."width", + "asset"."height" +from + "asset" + left join "asset_exif" on "asset"."id" = "asset_exif"."assetId" +where + "asset"."ownerId" = any ($1::uuid[]) + and true +order by + random() +limit + $2 + +-- SearchRepository.searchRandomV3 (with-filter) +select + "asset"."id", + "asset"."updateId", + "asset"."createdAt", + "asset"."updatedAt", + "asset"."deletedAt", + "asset"."status", + "asset"."checksum", + "asset"."checksumAlgorithm", + "asset"."duplicateId", + "asset"."duration", + "asset"."fileCreatedAt", + "asset"."fileModifiedAt", + "asset"."isExternal", + "asset"."isFavorite", + "asset"."isOffline", + "asset"."isEdited", + "asset"."visibility", + "asset"."libraryId", + "asset"."livePhotoVideoId", + "asset"."localDateTime", + "asset"."originalFileName", + "asset"."originalPath", + "asset"."ownerId", + "asset"."stackId", + "asset"."thumbhash", + "asset"."type", + "asset"."width", + "asset"."height" +from + "asset" + left join "asset_exif" on "asset"."id" = "asset_exif"."assetId" +where + "asset"."ownerId" = any ($1::uuid[]) + and "asset"."isFavorite" = $2 +order by + random() +limit + $3 + +-- SearchRepository.searchSmartV3 (baseline) +begin +set + local vchordrq.probes = 1 +select + "asset"."id", + "asset"."updateId", + "asset"."createdAt", + "asset"."updatedAt", + "asset"."deletedAt", + "asset"."status", + "asset"."checksum", + "asset"."checksumAlgorithm", + "asset"."duplicateId", + "asset"."duration", + "asset"."fileCreatedAt", + "asset"."fileModifiedAt", + "asset"."isExternal", + "asset"."isFavorite", + "asset"."isOffline", + "asset"."isEdited", + "asset"."visibility", + "asset"."libraryId", + "asset"."livePhotoVideoId", + "asset"."localDateTime", + "asset"."originalFileName", + "asset"."originalPath", + "asset"."ownerId", + "asset"."stackId", + "asset"."thumbhash", + "asset"."type", + "asset"."width", + "asset"."height" +from + "asset" + left join "asset_exif" on "asset"."id" = "asset_exif"."assetId" + inner join "smart_search" on "asset"."id" = "smart_search"."assetId" +where + "asset"."ownerId" = any ($1::uuid[]) + and true +order by + smart_search.embedding <=> $2, + "asset"."id" asc +limit + $3 +offset + $4 +commit + +-- SearchRepository.searchSmartV3 (with-filter) +begin +set + local vchordrq.probes = 1 +select + "asset"."id", + "asset"."updateId", + "asset"."createdAt", + "asset"."updatedAt", + "asset"."deletedAt", + "asset"."status", + "asset"."checksum", + "asset"."checksumAlgorithm", + "asset"."duplicateId", + "asset"."duration", + "asset"."fileCreatedAt", + "asset"."fileModifiedAt", + "asset"."isExternal", + "asset"."isFavorite", + "asset"."isOffline", + "asset"."isEdited", + "asset"."visibility", + "asset"."libraryId", + "asset"."livePhotoVideoId", + "asset"."localDateTime", + "asset"."originalFileName", + "asset"."originalPath", + "asset"."ownerId", + "asset"."stackId", + "asset"."thumbhash", + "asset"."type", + "asset"."width", + "asset"."height" +from + "asset" + left join "asset_exif" on "asset"."id" = "asset_exif"."assetId" + inner join "smart_search" on "asset"."id" = "smart_search"."assetId" +where + "asset"."ownerId" = any ($1::uuid[]) + and ( + "asset"."fileCreatedAt" < $2 + and "asset"."fileCreatedAt" >= $3 + ) +order by + smart_search.embedding <=> $4, + "asset"."id" asc +limit + $5 +offset + $6 +commit + +-- SearchRepository.searchSmartV3 (cursor-offset) +begin +set + local vchordrq.probes = 1 +select + "asset"."id", + "asset"."updateId", + "asset"."createdAt", + "asset"."updatedAt", + "asset"."deletedAt", + "asset"."status", + "asset"."checksum", + "asset"."checksumAlgorithm", + "asset"."duplicateId", + "asset"."duration", + "asset"."fileCreatedAt", + "asset"."fileModifiedAt", + "asset"."isExternal", + "asset"."isFavorite", + "asset"."isOffline", + "asset"."isEdited", + "asset"."visibility", + "asset"."libraryId", + "asset"."livePhotoVideoId", + "asset"."localDateTime", + "asset"."originalFileName", + "asset"."originalPath", + "asset"."ownerId", + "asset"."stackId", + "asset"."thumbhash", + "asset"."type", + "asset"."width", + "asset"."height" +from + "asset" + left join "asset_exif" on "asset"."id" = "asset_exif"."assetId" + inner join "smart_search" on "asset"."id" = "smart_search"."assetId" +where + "asset"."ownerId" = any ($1::uuid[]) + and true +order by + smart_search.embedding <=> $2, + "asset"."id" asc +limit + $3 +offset + $4 +commit -- SearchRepository.searchStatisticsV3 (baseline) select diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 70ba09ef2c..fe6f05703b 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -14,6 +14,8 @@ import { searchAssetBuilder, searchAssetBuilderLegacy, searchMetadataV3Examples, + searchRandomV3Examples, + searchSmartV3Examples, searchStatisticsV3Examples, withExifInner, withSearchOrder, @@ -146,6 +148,7 @@ export interface AssetSearchBuilderV3Options { export interface AssetSearchPaginationV3Options { size: number; + offset?: number; } export type SmartSearchOptions = SearchDateOptions & @@ -208,6 +211,7 @@ export interface GetCameraLensModelsOptions { export class SearchRepository { constructor(@InjectKysely() private db: Kysely) {} + // TODO(v4): remove with the deprecated flat-field search API @GenerateSql({ params: [ { page: 1, size: 100 }, @@ -233,6 +237,7 @@ export class SearchRepository { return paginationHelper(items, pagination.size); } + // TODO(v4): remove with the deprecated flat-field search API @GenerateSql({ params: [ { @@ -249,6 +254,7 @@ export class SearchRepository { .executeTakeFirstOrThrow(); } + // TODO(v4): remove with the deprecated flat-field search API @GenerateSql({ params: [ 100, @@ -269,6 +275,7 @@ export class SearchRepository { .execute(); } + // TODO(v4): remove with the deprecated flat-field search API @GenerateSql({ params: [ 100, @@ -292,6 +299,7 @@ export class SearchRepository { .execute(); } + // TODO(v4): remove with the deprecated flat-field search API @GenerateSql({ params: [ { page: 1, size: 200 }, @@ -518,17 +526,48 @@ export class SearchRepository { return res.map((row) => row.lensModel!); } + // TODO(v4): drop the V3 suffix once the legacy methods are removed @GenerateSql(...searchMetadataV3Examples) - searchMetadataV3( - pagination: AssetSearchPaginationV3Options, - options: AssetSearchBuilderV3Options, - ): Promise { - return withSearchOrder(searchAssetBuilder(this.db, options), options.order) + async searchMetadataV3(pagination: AssetSearchPaginationV3Options, options: AssetSearchBuilderV3Options) { + const items = await withSearchOrder(searchAssetBuilder(this.db, options), options.order) .select(columns.searchAsset) - .limit(pagination.size) + .limit(pagination.size + 1) + .offset(pagination.offset ?? 0) + .execute(); + return paginationHelper(items, pagination.size); + } + + // TODO(v4): drop the V3 suffix once the legacy methods are removed + @GenerateSql(...searchRandomV3Examples) + searchRandomV3(size: number, options: Omit): Promise { + return searchAssetBuilder(this.db, options) + .select(columns.searchAsset) + .orderBy(sql`random()`) + .limit(size) .execute(); } + // TODO(v4): drop the V3 suffix once the legacy methods are removed + @GenerateSql(...searchSmartV3Examples) + searchSmartV3( + pagination: AssetSearchPaginationV3Options, + options: Omit & { embedding: string }, + ) { + return this.db.transaction().execute(async (trx) => { + await sql`set local vchordrq.probes = ${sql.lit(probes[VectorIndex.Clip])}`.execute(trx); + const items = await searchAssetBuilder(trx, options) + .select(columns.searchAsset) + .innerJoin('smart_search', 'asset.id', 'smart_search.assetId') + .orderBy(sql`smart_search.embedding <=> ${options.embedding}`) + .orderBy('asset.id', 'asc') + .limit(pagination.size + 1) + .offset(pagination.offset ?? 0) + .execute(); + return paginationHelper(items, pagination.size); + }); + } + + // TODO(v4): drop the V3 suffix once the legacy methods are removed @GenerateSql(...searchStatisticsV3Examples) searchStatisticsV3(options: AssetSearchBuilderV3Options) { return searchAssetBuilder(this.db, options) diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index 0f4d8775b6..960ad59bfe 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -938,6 +938,37 @@ export const searchMetadataV3Examples: GenerateSqlQueries[] = [ }, ], }, + { + name: 'cursor-offset', + params: [{ size: 100, offset: 100 }, { userIds: [DummyValue.UUID], filter: { isFavorite: { eq: true } } }], + }, +]; + +export const searchRandomV3Examples: GenerateSqlQueries[] = [ + { name: 'baseline', params: [100, { userIds: [DummyValue.UUID] }] }, + { + name: 'with-filter', + params: [100, { userIds: [DummyValue.UUID], filter: { isFavorite: { eq: true } } }], + }, +]; + +export const searchSmartV3Examples: GenerateSqlQueries[] = [ + { name: 'baseline', params: [{ size: 100 }, { userIds: [DummyValue.UUID], embedding: DummyValue.VECTOR }] }, + { + name: 'with-filter', + params: [ + { size: 100 }, + { + userIds: [DummyValue.UUID], + embedding: DummyValue.VECTOR, + filter: { takenAt: { gte: DummyValue.DATE, lt: DummyValue.DATE } }, + }, + ], + }, + { + name: 'cursor-offset', + params: [{ size: 100, offset: 100 }, { userIds: [DummyValue.UUID], embedding: DummyValue.VECTOR }], + }, ]; export const searchStatisticsV3Examples: GenerateSqlQueries[] = [