diff --git a/README.md b/README.md index c17d05fe..54a59715 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ volumes: - `./public` - The folder where all the public assets are stored (must mount to `/zipline/public`) - `./themes` - The folder where all the custom themes are stored (must mount to `/zipline/themes`) +Temporary files default to `./uploads/.tmp`. Setting `CORE_TEMP_DIRECTORY` to another filesystem, such as tmpfs, can reduce local upload performance. + ### Generating Secrets ```bash diff --git a/prisma/migrations/20260724223042_add_file_user_id_index/migration.sql b/prisma/migrations/20260724223042_add_file_user_id_index/migration.sql new file mode 100644 index 00000000..bb2492f3 --- /dev/null +++ b/prisma/migrations/20260724223042_add_file_user_id_index/migration.sql @@ -0,0 +1,2 @@ +-- CreateIndex +CREATE INDEX "File_userId_size_idx" ON "public"."File"("userId", "size"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index af519618..8986c2de 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -17,7 +17,7 @@ model Zipline { coreReturnHttpsUrls Boolean @default(false) coreDefaultDomain String? - coreTempDirectory String // default join(tmpdir(), 'zipline') + coreTempDirectory String // default resolve('./uploads/.tmp') coreTrustProxy Boolean @default(false) chunksEnabled Boolean @default(true) @@ -298,6 +298,7 @@ model File { thumbnail Thumbnail? @@index([name]) + @@index([userId, size]) @@index([folderId, createdAt]) } diff --git a/src/lib/api/upload.ts b/src/lib/api/upload.ts index 6c044410..eec9cdc9 100644 --- a/src/lib/api/upload.ts +++ b/src/lib/api/upload.ts @@ -33,30 +33,26 @@ export function getExtension(filename: string, override?: string) { } export async function checkQuota( - user: User | null, + user: Pick | null, newSize: number, fileCount: number, ): Promise { if (!user?.quota) return true; - const stats = await prisma.file.aggregate({ - where: { - userId: user.id, - }, - _sum: { - size: true, - }, - _count: { - _all: true, - }, - }); + if (user.quota.filesQuota === 'BY_BYTES') { + const stats = await prisma.file.aggregate({ + where: { userId: user.id }, + _sum: { size: true }, + }); - const aggSize = stats?._sum?.size ? stats._sum.size : 0n; + if (Number(stats._sum.size ?? 0n) + newSize > bytes(user.quota.maxBytes!)) + return `uploading will exceed your storage quota of ${user.quota.maxBytes}`; - if (user.quota.filesQuota === 'BY_BYTES' && Number(aggSize) + newSize > bytes(user.quota.maxBytes!)) - return `uploading will exceed your storage quota of ${user.quota.maxFiles} files`; + return true; + } - if (user.quota.filesQuota === 'BY_FILES' && stats?._count?._all + fileCount > user.quota.maxFiles!) + const count = await prisma.file.count({ where: { userId: user.id } }); + if (count + fileCount > user.quota.maxFiles!) return `uploading will exceed your file count quota of ${user.quota.maxFiles} files`; return true; @@ -81,6 +77,7 @@ export async function getFilename( originalName: string, extension: string, override?: string, + reservedNames?: Set, ): Promise<{ error: string } | { fileName: string }> { try { let fileName = override ? sanitizeFilename(override) : formatFileName(format, originalName); @@ -88,7 +85,8 @@ export async function getFilename( if (!fileName) return { error: 'invalid file name' }; let fullFileName = `${fileName}${extension}`; - let existing = await prisma.file.findFirst({ where: { name: fullFileName } }); + let existing = + reservedNames?.has(fullFileName) || (await prisma.file.findFirst({ where: { name: fullFileName } })); if (existing && (override || format === 'name')) { return { error: 'file with the same name already exists' }; @@ -101,9 +99,11 @@ export async function getFilename( if (!fileName) return { error: 'invalid file name' }; fullFileName = `${fileName}${extension}`; - existing = await prisma.file.findFirst({ where: { name: fullFileName } }); + existing = + reservedNames?.has(fullFileName) || (await prisma.file.findFirst({ where: { name: fullFileName } })); } + reservedNames?.add(fullFileName); return { fileName }; } catch (e) { logger.warn(`error generating file name: ${e}`); diff --git a/src/lib/config/read/db.ts b/src/lib/config/read/db.ts index c3542c95..6262184f 100644 --- a/src/lib/config/read/db.ts +++ b/src/lib/config/read/db.ts @@ -1,6 +1,5 @@ import { prisma } from '@/lib/db'; -import { tmpdir } from 'os'; -import { join } from 'path'; +import { resolve } from 'path'; export const DATABASE_TO_PROP = { coreReturnHttpsUrls: 'core.returnHttpsUrls', @@ -154,7 +153,7 @@ export async function readDatabaseSettings() { if (!ziplineTable) { ziplineTable = await prisma.zipline.create({ data: { - coreTempDirectory: join(tmpdir(), 'zipline'), + coreTempDirectory: resolve('./uploads/.tmp'), }, omit: { createdAt: true, diff --git a/src/lib/config/validate.ts b/src/lib/config/validate.ts index ee0895a5..767aaa22 100644 --- a/src/lib/config/validate.ts +++ b/src/lib/config/validate.ts @@ -1,5 +1,4 @@ -import { tmpdir } from 'os'; -import { join, resolve } from 'path'; +import { resolve } from 'path'; import { z } from 'zod'; import { log } from '../logger'; import { ParsedConfig } from './read'; @@ -99,7 +98,7 @@ export const schema = z.object({ tempDirectory: z .string() .transform((s) => resolve(s)) - .default(join(tmpdir(), 'zipline')), + .default(resolve('./uploads/.tmp')), trustProxy: z.boolean().default(false), databaseUrl: z.url(), diff --git a/src/lib/datasource/Local.ts b/src/lib/datasource/Local.ts index 6e0992e1..1be868cb 100644 --- a/src/lib/datasource/Local.ts +++ b/src/lib/datasource/Local.ts @@ -101,15 +101,15 @@ export class LocalDatasource extends Datasource { } public async totalSize(): Promise { - const files = await readdir(this.dir); - const sizes = await Promise.all(files.map((file) => this.size(file))); + const files = (await readdir(this.dir, { withFileTypes: true })).filter((file) => file.isFile()); + const sizes = await Promise.all(files.map((file) => this.size(file.name))); return sizes.reduce((a, b) => a + b, 0); } public async clear(): Promise { - for (const file of await readdir(this.dir)) { - await rm(join(this.dir, file)); + for (const file of await readdir(this.dir, { withFileTypes: true })) { + if (file.isFile()) await rm(join(this.dir, file.name)); } } diff --git a/src/lib/datasource/S3.ts b/src/lib/datasource/S3.ts index d6c6c4c9..f215f369 100644 --- a/src/lib/datasource/S3.ts +++ b/src/lib/datasource/S3.ts @@ -13,7 +13,9 @@ import { UploadPartCopyCommand, } from '@aws-sdk/client-s3'; import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { Upload } from '@aws-sdk/lib-storage'; import { createReadStream } from 'fs'; +import { stat } from 'fs/promises'; import { Agent as HttpAgent } from 'http'; import { Agent as HttpsAgent } from 'https'; import { Readable } from 'stream'; @@ -168,36 +170,49 @@ export class S3Datasource extends Datasource { } public async put(file: string, data: Buffer | string, options: PutOptions = {}): Promise { - let command = new PutObjectCommand({ - Bucket: this.options.bucket, - Key: this.key(file), - Body: data, - ...(options.mimetype ? { ContentType: options.mimetype } : {}), - }); + try { + if (typeof data === 'string') { + const size = await stat(data).then((file) => file.size); + if (size > 25 * 1024 * 1024) { + // 25mb + this.logger.debug('putting object with multipart upload', { file, key: this.key(file) }); - if (typeof data === 'string') { - const readStream = createReadStream(data); - command = new PutObjectCommand({ + try { + const upload = new Upload({ + client: this.client, + params: { + Bucket: this.options.bucket, + Key: this.key(file), + Body: createReadStream(data), + ...(options.mimetype ? { ContentType: options.mimetype } : {}), + }, + leavePartsOnError: false, + }); + + await upload.done(); + return; + } catch (error) { + this.logger.warn('multipart upload failed, retrying with a single request', { + error: error instanceof Error ? error.message : error, + }); + } + } + } + + const command = new PutObjectCommand({ Bucket: this.options.bucket, Key: this.key(file), - Body: readStream, + Body: typeof data === 'string' ? createReadStream(data) : data, ...(options.mimetype ? { ContentType: options.mimetype } : {}), }); - - this.logger.debug('putting object from stream', { file, key: this.key(file) }); - } - - try { const res = await this.client.send(command); if (!isOk(res.$metadata.httpStatusCode || 0)) { - this.logger.error( - 'there was an error while putting object', - res.$metadata as Record, - ); + throw new Error(`S3 put failed with status ${res.$metadata.httpStatusCode ?? 'unknown'}`); } } catch (e) { this.logger.error('there was an error while putting object', e as Record); + throw e; } } diff --git a/src/lib/db/models/zipline.ts b/src/lib/db/models/zipline.ts index 7d5ef849..11e8e230 100644 --- a/src/lib/db/models/zipline.ts +++ b/src/lib/db/models/zipline.ts @@ -1,14 +1,12 @@ -import { tmpdir } from 'os'; import { prisma } from '..'; -import { join } from 'path'; +import { resolve } from 'path'; export async function getZipline() { const zipline = await prisma.zipline.findFirst(); if (!zipline) { - const tmp = join(tmpdir(), 'zipline'); return prisma.zipline.create({ data: { - coreTempDirectory: tmp, + coreTempDirectory: resolve('./uploads/.tmp'), }, }); } diff --git a/src/lib/mapConcurrent.ts b/src/lib/mapConcurrent.ts new file mode 100644 index 00000000..0d02e74d --- /dev/null +++ b/src/lib/mapConcurrent.ts @@ -0,0 +1,27 @@ +export async function mapConcurrent( + values: T[], + concurrency: number, + map: (value: T, index: number) => Promise, +): Promise { + const results = Array(values.length); + let next = 0; + let failed = false; + let failure: unknown; + + async function worker() { + while (!failed && next < values.length) { + const index = next++; + try { + results[index] = await map(values[index], index); + } catch (error) { + failed = true; + failure = error; + } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, worker)); + if (failed) throw failure; + + return results; +} diff --git a/src/lib/mimes.ts b/src/lib/mimes.ts index af6093fd..6ab31dad 100644 --- a/src/lib/mimes.ts +++ b/src/lib/mimes.ts @@ -2,13 +2,12 @@ import { readFile } from 'fs/promises'; export type Mimes = [string, string[]][]; +const mimes = readFile(new URL('../../mimes.json', import.meta.url), 'utf8').then( + (data) => new Map((JSON.parse(data) as Mimes).map(([extension, types]) => [extension, types[0]])), +); + export async function guess(extension: string | null): Promise { if (!extension) return 'application/octet-stream'; - const mimes: Mimes = JSON.parse(await readFile('./mimes.json', 'utf8')); - - const mime = mimes.find((x) => x[0] === extension); - if (!mime) return 'application/octet-stream'; - - return mime[1][0]; + return (await mimes).get(extension) ?? 'application/octet-stream'; } diff --git a/src/lib/webhooks/index.ts b/src/lib/webhooks/index.ts index 631e7c1f..320c567c 100644 --- a/src/lib/webhooks/index.ts +++ b/src/lib/webhooks/index.ts @@ -1,15 +1,18 @@ import { Config } from '../config/validate'; +import { log } from '../logger'; import { onUpload as discordOnUpload, onShorten as discordOnShorten } from './discord'; import { onUpload as httpOnUpload, onShorten as httpOnShorten } from './http'; -export async function onUpload(config: Config, args: Parameters[1]) { - Promise.all([discordOnUpload(config, args), httpOnUpload(config, args)]); +const logger = log('webhooks'); - return; +export function onUpload(config: Config, args: Parameters[1]) { + void Promise.all([discordOnUpload(config, args), httpOnUpload(config, args)]).catch((error) => + logger.error('upload webhook failed', { error: error instanceof Error ? error.message : error }), + ); } -export async function onShorten(config: Config, args: Parameters[1]) { - Promise.all([discordOnShorten(config, args), httpOnShorten(config, args)]); - - return; +export function onShorten(config: Config, args: Parameters[1]) { + void Promise.all([discordOnShorten(config, args), httpOnShorten(config, args)]).catch((error) => + logger.error('shorten webhook failed', { error: error instanceof Error ? error.message : error }), + ); } diff --git a/src/offload/partial.ts b/src/offload/partial.ts index d5bc133a..27846c24 100644 --- a/src/offload/partial.ts +++ b/src/offload/partial.ts @@ -4,7 +4,7 @@ import { getDatasource } from '@/lib/datasource'; import { S3Datasource } from '@/lib/datasource/S3'; import { File, fileSelect } from '@/lib/db/models/file'; import { IncompleteFile } from '@/lib/db/models/incompleteFile'; -import { User, userSelect } from '@/lib/db/models/user'; +import { limitedUserSelect, User, userSelect } from '@/lib/db/models/user'; import { log } from '@/lib/logger'; import { randomCharacters } from '@/lib/random'; import { UploadOptions } from '@/lib/uploader/parseHeaders'; @@ -192,7 +192,7 @@ async function runComplete(id: string, size: number) { where: { id: user.id, }, - select: userSelect, + select: config.discord?.onUpload || config.httpWebhook.onUpload ? userSelect : limitedUserSelect, }); if (!userr) return; diff --git a/src/server/middleware/user.ts b/src/server/middleware/user.ts index 82080d4c..db9b8837 100644 --- a/src/server/middleware/user.ts +++ b/src/server/middleware/user.ts @@ -1,7 +1,7 @@ import { config } from '@/lib/config'; import { decryptToken } from '@/lib/crypto'; import { prisma } from '@/lib/db'; -import { User, userSelect } from '@/lib/db/models/user'; +import { limitedUserSelect, User, userSelect } from '@/lib/db/models/user'; import { FastifyReply } from 'fastify'; import { FastifyRequest } from 'fastify/types/request'; import { getSession } from '../session'; @@ -41,10 +41,14 @@ export function parseUserToken( } export async function userMiddleware(req: FastifyRequest, res: FastifyReply) { + const path = req.url.toLowerCase().split('?')[0]; + const upload = ['/api/upload', '/api/upload/partial'].includes(path); + const leanUpload = upload && !config.discord?.onUpload && !config.httpWebhook.onUpload; + // conditions met to allow anonymous folder uploads but later handled in the upload route const anonFolderUpload = req.headers['x-zipline-folder'] && - ['/api/upload', '/api/upload/partial'].includes(req.url.toLowerCase().split('?')[0]) && + upload && !req.headers.authorization && !req.cookies['zipline_session']; if (anonFolderUpload) return; @@ -58,11 +62,11 @@ export async function userMiddleware(req: FastifyRequest, res: FastifyReply) { where: { token, }, - select: userSelect, + select: leanUpload ? limitedUserSelect : userSelect, }); if (!user) throw new ApiError(2001); - req.user = user; + req.user = user as User; return; } @@ -80,9 +84,9 @@ export async function userMiddleware(req: FastifyRequest, res: FastifyReply) { }, }, }, - select: userSelect, + select: leanUpload ? limitedUserSelect : userSelect, }); if (!user) throw new ApiError(2001); - req.user = user; + req.user = user as User; } diff --git a/src/server/routes/api/upload/index.ts b/src/server/routes/api/upload/index.ts index 6b5533d1..02e46468 100644 --- a/src/server/routes/api/upload/index.ts +++ b/src/server/routes/api/upload/index.ts @@ -11,6 +11,7 @@ import { userSelect } from '@/lib/db/models/user'; import { sanitizeFilename } from '@/lib/fs'; import { removeGps } from '@/lib/gps'; import { log } from '@/lib/logger'; +import { mapConcurrent } from '@/lib/mapConcurrent'; import { runThumbnailWorkers } from '@/lib/tasks/run/thumbnails'; import { parseHeaders, UploadHeaders } from '@/lib/uploader/parseHeaders'; import { onUpload } from '@/lib/webhooks'; @@ -18,7 +19,6 @@ import { Prisma } from '@/prisma/client'; import { userMiddleware } from '@/server/middleware/user'; import typedPlugin from '@/server/typedPlugin'; import { SavedMultipartFile } from '@fastify/multipart'; -import { stat } from 'fs/promises'; import { z } from 'zod'; export type ApiUploadResponse = { @@ -29,7 +29,7 @@ export type ApiUploadResponse = { url: string; pending?: boolean; removedGps?: boolean; - compressed?: CompressResult; + compressed?: Omit; }[]; deletesAt?: string; @@ -135,7 +135,7 @@ export default typedPlugin( ...(options.deletesAt && { deletesAt: options.deletesAt === 'never' ? 'never' : options.deletesAt.toISOString(), }), - ...(config.files.assumeMimetypes && { assumedMimetypes: Array(req.files.length) }), + ...(config.files.assumeMimetypes && { assumedMimetypes: Array(files.length) }), }; const domain = getDomain( @@ -146,8 +146,8 @@ export default typedPlugin( logger.debug('uploading files', { files: files.map((x) => x.filename) }); - for (let i = 0; i !== files.length; ++i) { - const file = files[i]; + // todo: maybe make configurable? + const prepared = await mapConcurrent(files, 4, async (file, i) => { const extension = getExtension(file.filename, options.overrides?.extension); if (config.files.disabledExtensions.includes(extension)) @@ -158,13 +158,6 @@ export default typedPlugin( `file[${i}]: File size is too large. Maximum file size is ${bytes(config.files.maxFileSize)} bytes`, ); - // determine filename - const format = options.format || config.files.defaultFormat; - const nameResult = await getFilename(format, file.filename, extension, options.overrides?.filename); - if ('error' in nameResult) throw new ApiError(1009, `file[${i}]: ${nameResult.error}`); - - const { fileName } = nameResult; - // determine mimetype const { assumed, ...mimeRes } = await getMimetype(file.mimetype, extension); let mimetype = mimeRes.mimetype; @@ -183,7 +176,6 @@ export default typedPlugin( } if (config.files.disabledTypes.includes(mimetype.trim().toLowerCase())) { - console.log(mimetype, config.files.disabledTypesDefault); if (config.files.disabledTypesDefault) mimetype = config.files.disabledTypesDefault; else throw new ApiError(1065, `file[${i}]: File type ${mimetype} is not allowed`); } @@ -207,18 +199,46 @@ export default typedPlugin( // remove gps metadata if requested let removedGps = false; if (mimetype.startsWith('image/') && config.files.removeGpsMetadata) { - const removed = removeGps(file.filepath); + const removed = removeGps(compressed?.buffer ?? file.filepath); if (removed) logger.c('gps').debug(`removed gps metadata from ${file.filename}`); removedGps = removed; } - const tempFileStats = await stat(file.filepath); + return { + file, + extension: compressed ? `.${compressed.ext}` : extension, + mimetype: compressed?.mimetype ?? mimetype, + size: compressed?.buffer.length ?? file.file.bytesRead, + compressed, + removedGps, + }; + }); + + const reservedNames = new Set(); + const format = options.format || config.files.defaultFormat; + const named: ((typeof prepared)[number] & { fileName: string })[] = []; + for (let i = 0; i < prepared.length; i++) { + const item = prepared[i]; + const nameResult = await getFilename( + format, + item.file.filename, + item.extension, + options.overrides?.filename, + reservedNames, + ); + if ('error' in nameResult) throw new ApiError(1009, `file[${i}]: ${nameResult.error}`); + + named.push({ ...item, fileName: nameResult.fileName }); + } + + response.files = await mapConcurrent(named, 4, async (item, i) => { + const { file, fileName, extension, mimetype, size, compressed, removedGps } = item; const data: Prisma.FileCreateInput = { - name: `${fileName}${compressed ? '.' + compressed.ext : extension}`, - size: compressed?.buffer?.length ?? tempFileStats.size, - type: compressed?.mimetype ?? mimetype, + name: `${fileName}${extension}`, + size, + type: mimetype, User: { connect: { id: req.user ? req.user.id : options.folder ? folder?.userId : undefined } }, }; @@ -241,24 +261,32 @@ export default typedPlugin( select: fileSelect, }); - await datasource.put(fileUpload.name, compressed?.buffer ?? file.filepath, { + const storageData = compressed?.buffer ?? file.filepath; + await datasource.put(fileUpload.name, storageData, { mimetype: fileUpload.type, }); + if (typeof storageData === 'string' && datasource.name === 'local' && req.tmpUploads) { + req.tmpUploads = req.tmpUploads.filter((path) => path !== storageData); + } const responseUrl = `${domain}${config.files.route === '/' || config.files.route === '' ? '' : `${config.files.route}`}/${fileUpload.name}`; - response.files.push({ + const compressedResponse = compressed + ? { mimetype: compressed.mimetype, ext: compressed.ext, failed: compressed.failed } + : undefined; + + const responseFile = { id: fileUpload.id, name: fileUpload.name, type: fileUpload.type, url: encodeURI(responseUrl), removedGps: removedGps || undefined, - compressed: compressed || undefined, - }); + compressed: compressedResponse, + }; logger.info( `${req.user ? req.user.username : '[anonymous folder upload]'} uploaded ${fileUpload.name}`, - { size: bytes(compressed?.buffer?.length ?? fileUpload.size), ip: req.ip }, + { size: bytes(fileUpload.size), ip: req.ip }, ); await onUpload(config, { @@ -275,7 +303,9 @@ export default typedPlugin( returned: encodeURI(responseUrl), }, }); - } + + return responseFile; + }); if (options.noJson) return res diff --git a/src/server/routes/api/upload/partial.ts b/src/server/routes/api/upload/partial.ts index ea471cf8..172c95b9 100644 --- a/src/server/routes/api/upload/partial.ts +++ b/src/server/routes/api/upload/partial.ts @@ -4,7 +4,7 @@ import { bytes } from '@/lib/bytes'; import { config } from '@/lib/config'; import { hashPassword } from '@/lib/crypto'; import { prisma } from '@/lib/db'; -import { userSelect } from '@/lib/db/models/user'; +import { limitedUserSelect } from '@/lib/db/models/user'; import { sanitizeFilename } from '@/lib/fs'; import { log } from '@/lib/logger'; import { guess } from '@/lib/mimes'; @@ -99,7 +99,7 @@ export default typedPlugin( ...(options.deletesAt && { deletesAt: options.deletesAt === 'never' ? 'never' : options.deletesAt.toISOString(), }), - ...(config.files.assumeMimetypes && { assumedMimetypes: Array(req.files.length) }), + ...(config.files.assumeMimetypes && { assumedMimetypes: Array(files.length) }), }; const domain = getDomain( @@ -144,7 +144,7 @@ export default typedPlugin( const quotaUser = req.user ? req.user : folder?.userId - ? await prisma.user.findUnique({ where: { id: folder.userId }, select: userSelect }) + ? await prisma.user.findUnique({ where: { id: folder.userId }, select: limitedUserSelect }) : null; // check quota, using the current added length, and only just adding one file @@ -175,6 +175,7 @@ export default typedPlugin( const tempFile = join(config.core.tempDirectory, sanitized); await rename(file.filepath, tempFile); + if (req.tmpUploads) req.tmpUploads = req.tmpUploads.filter((path) => path !== file.filepath); if (options.partial.lastchunk) { const extension = getExtension(options.partial.filename, options.overrides?.extension);