mirror of
https://github.com/immich-app/immich.git
synced 2026-01-15 06:23:00 -08:00
* duplicate detection job, entity, config * queueing * job panel, update api * use embedding in db instead of fetching * disable concurrency * only queue visible assets * handle multiple duplicateIds * update concurrent queue check * add provider * add web placeholder, server endpoint, migration, various fixes * update sql * select embedding by default * rename variable * simplify * remove separate entity, handle re-running with different threshold, set default back to 0.02 * fix tests * add tests * add index to entity * formatting * update asset mock * fix `upsertJobStatus` signature * update sql * formatting * default to 0.03 * optimize clustering * use asset's `duplicateId` if present * update sql * update tests * expose admin setting * refactor * formatting * skip if ml is disabled * debug trash e2e * remove from web * remove from sidebar * test if ml is disabled * update sql * separate duplicate detection from clip in config, disable by default for now * fix doc * lower minimum `maxDistance` * update api * Add and Use Duplicate Detection Feature Flag (#9364) * Add Duplicate Detection Flag * Use Duplicate Detection Flag * Attempt Fixes for Failing Checks * lower minimum `maxDistance` * fix tests --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> * chore: fixes and additions after rebase * chore: update api (remove new Role enum) * fix: left join smart search so getAll works without machine learning * test: trash e2e go back to checking length of assets is zero * chore: regen api after rebase * test: fix tests after rebase * redundant join --------- Co-authored-by: Nicholas Flamy <30300649+NicholasFlamy@users.noreply.github.com> Co-authored-by: Zack Pollard <zackpollard@ymail.com> Co-authored-by: Zack Pollard <zack@futo.org>
184 lines
4.3 KiB
TypeScript
184 lines
4.3 KiB
TypeScript
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
|
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
|
|
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
|
|
import { Paginated } from 'src/utils/pagination';
|
|
|
|
export const ISearchRepository = 'ISearchRepository';
|
|
|
|
export interface SearchResult<T> {
|
|
/** total matches */
|
|
total: number;
|
|
/** collection size */
|
|
count: number;
|
|
/** current page */
|
|
page: number;
|
|
/** items for page */
|
|
items: T[];
|
|
/** score */
|
|
distances: number[];
|
|
facets: SearchFacet[];
|
|
}
|
|
|
|
export interface SearchFacet {
|
|
fieldName: string;
|
|
counts: Array<{
|
|
count: number;
|
|
value: string;
|
|
}>;
|
|
}
|
|
|
|
export type SearchExploreItemSet<T> = Array<{
|
|
value: string;
|
|
data: T;
|
|
}>;
|
|
|
|
export interface SearchExploreItem<T> {
|
|
fieldName: string;
|
|
items: SearchExploreItemSet<T>;
|
|
}
|
|
|
|
export type Embedding = number[];
|
|
|
|
export interface SearchAssetIDOptions {
|
|
checksum?: Buffer;
|
|
deviceAssetId?: string;
|
|
id?: string;
|
|
}
|
|
|
|
export interface SearchUserIdOptions {
|
|
deviceId?: string;
|
|
libraryId?: string;
|
|
userIds?: string[];
|
|
}
|
|
|
|
export type SearchIdOptions = SearchAssetIDOptions & SearchUserIdOptions;
|
|
|
|
export interface SearchStatusOptions {
|
|
isArchived?: boolean;
|
|
isEncoded?: boolean;
|
|
isFavorite?: boolean;
|
|
isMotion?: boolean;
|
|
isOffline?: boolean;
|
|
isVisible?: boolean;
|
|
isNotInAlbum?: boolean;
|
|
type?: AssetType;
|
|
withArchived?: boolean;
|
|
withDeleted?: boolean;
|
|
}
|
|
|
|
export interface SearchOneToOneRelationOptions {
|
|
withExif?: boolean;
|
|
withSmartInfo?: boolean;
|
|
withStacked?: boolean;
|
|
}
|
|
|
|
export interface SearchRelationOptions extends SearchOneToOneRelationOptions {
|
|
withFaces?: boolean;
|
|
withPeople?: boolean;
|
|
}
|
|
|
|
export interface SearchDateOptions {
|
|
createdBefore?: Date;
|
|
createdAfter?: Date;
|
|
takenBefore?: Date;
|
|
takenAfter?: Date;
|
|
trashedBefore?: Date;
|
|
trashedAfter?: Date;
|
|
updatedBefore?: Date;
|
|
updatedAfter?: Date;
|
|
}
|
|
|
|
export interface SearchPathOptions {
|
|
encodedVideoPath?: string;
|
|
originalFileName?: string;
|
|
originalPath?: string;
|
|
previewPath?: string;
|
|
thumbnailPath?: string;
|
|
}
|
|
|
|
export interface SearchExifOptions {
|
|
city?: string;
|
|
country?: string;
|
|
lensModel?: string;
|
|
make?: string;
|
|
model?: string;
|
|
state?: string;
|
|
}
|
|
|
|
export interface SearchEmbeddingOptions {
|
|
embedding: Embedding;
|
|
userIds: string[];
|
|
}
|
|
|
|
export interface SearchPeopleOptions {
|
|
personIds?: string[];
|
|
}
|
|
|
|
export interface SearchOrderOptions {
|
|
orderDirection?: 'ASC' | 'DESC';
|
|
}
|
|
|
|
export interface SearchPaginationOptions {
|
|
page: number;
|
|
size: number;
|
|
}
|
|
|
|
type BaseAssetSearchOptions = SearchDateOptions &
|
|
SearchIdOptions &
|
|
SearchExifOptions &
|
|
SearchOrderOptions &
|
|
SearchPathOptions &
|
|
SearchStatusOptions &
|
|
SearchUserIdOptions &
|
|
SearchPeopleOptions;
|
|
|
|
export type AssetSearchOptions = BaseAssetSearchOptions & SearchRelationOptions;
|
|
|
|
export type AssetSearchOneToOneRelationOptions = BaseAssetSearchOptions & SearchOneToOneRelationOptions;
|
|
|
|
export type AssetSearchBuilderOptions = Omit<AssetSearchOptions, 'orderDirection'>;
|
|
|
|
export type SmartSearchOptions = SearchDateOptions &
|
|
SearchEmbeddingOptions &
|
|
SearchExifOptions &
|
|
SearchOneToOneRelationOptions &
|
|
SearchStatusOptions &
|
|
SearchUserIdOptions &
|
|
SearchPeopleOptions;
|
|
|
|
export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
|
|
hasPerson?: boolean;
|
|
numResults: number;
|
|
maxDistance?: number;
|
|
}
|
|
|
|
export interface AssetDuplicateSearch {
|
|
assetId: string;
|
|
embedding: Embedding;
|
|
userIds: string[];
|
|
maxDistance?: number;
|
|
}
|
|
|
|
export interface FaceSearchResult {
|
|
distance: number;
|
|
face: AssetFaceEntity;
|
|
}
|
|
|
|
export interface AssetDuplicateResult {
|
|
assetId: string;
|
|
duplicateId: string | null;
|
|
distance: number;
|
|
}
|
|
|
|
export interface ISearchRepository {
|
|
init(modelName: string): Promise<void>;
|
|
searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity>;
|
|
searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated<AssetEntity>;
|
|
searchDuplicates(options: AssetDuplicateSearch): Promise<AssetDuplicateResult[]>;
|
|
searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
|
|
upsert(assetId: string, embedding: number[]): Promise<void>;
|
|
searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]>;
|
|
getAssetsByCity(userIds: string[]): Promise<AssetEntity[]>;
|
|
deleteAllSearchEmbeddings(): Promise<void>;
|
|
}
|