feat: speed up file uploads (#1121)

2026-07-28 12:56:59 -07:00 · 2026-07-27 17:19:14 -07:00
parent 6cb93e6719
commit 1db8174917
16 changed files with 180 additions and 100 deletions
@@ -100,6 +100,8 @@ volumes:
 - `./public` - The folder where all the public assets are stored (must mount to `/zipline/public`)
 - `./themes` - The folder where all the custom themes are stored (must mount to `/zipline/themes`)

+Temporary files default to `./uploads/.tmp`. Setting `CORE_TEMP_DIRECTORY` to another filesystem, such as tmpfs, can reduce local upload performance.
+
 ### Generating Secrets

 ```bash
@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX "File_userId_size_idx" ON "public"."File"("userId", "size");
@@ -17,7 +17,7 @@ model Zipline {

  coreReturnHttpsUrls Boolean @default(false)
  coreDefaultDomain   String?
-  coreTempDirectory   String // default join(tmpdir(), 'zipline')
+  coreTempDirectory   String // default resolve('./uploads/.tmp')
  coreTrustProxy      Boolean @default(false)

  chunksEnabled Boolean @default(true)
@@ -298,6 +298,7 @@ model File {
  thumbnail Thumbnail?

  @@index([name])
+  @@index([userId, size])
  @@index([folderId, createdAt])
 }

@@ -33,30 +33,26 @@ export function getExtension(filename: string, override?: string) {
 }

 export async function checkQuota(
-  user: User | null,
+  user: Pick<User, 'id' | 'quota'> | null,
  newSize: number,
  fileCount: number,
 ): Promise<true | string> {
  if (!user?.quota) return true;

-  const stats = await prisma.file.aggregate({
-    where: {
-      userId: user.id,
-    },
-    _sum: {
-      size: true,
-    },
-    _count: {
-      _all: true,
-    },
-  });
+  if (user.quota.filesQuota === 'BY_BYTES') {
+    const stats = await prisma.file.aggregate({
+      where: { userId: user.id },
+      _sum: { size: true },
+    });

-  const aggSize = stats?._sum?.size ? stats._sum.size : 0n;
+    if (Number(stats._sum.size ?? 0n) + newSize > bytes(user.quota.maxBytes!))
+      return `uploading will exceed your storage quota of ${user.quota.maxBytes}`;

-  if (user.quota.filesQuota === 'BY_BYTES' && Number(aggSize) + newSize > bytes(user.quota.maxBytes!))
-    return `uploading will exceed your storage quota of ${user.quota.maxFiles} files`;
+    return true;
+  }

-  if (user.quota.filesQuota === 'BY_FILES' && stats?._count?._all + fileCount > user.quota.maxFiles!)
+  const count = await prisma.file.count({ where: { userId: user.id } });
+  if (count + fileCount > user.quota.maxFiles!)
    return `uploading will exceed your file count quota of ${user.quota.maxFiles} files`;

  return true;
@@ -81,6 +77,7 @@ export async function getFilename(
  originalName: string,
  extension: string,
  override?: string,
+  reservedNames?: Set<string>,
 ): Promise<{ error: string } | { fileName: string }> {
  try {
    let fileName = override ? sanitizeFilename(override) : formatFileName(format, originalName);
@@ -88,7 +85,8 @@ export async function getFilename(
    if (!fileName) return { error: 'invalid file name' };

    let fullFileName = `${fileName}${extension}`;
-    let existing = await prisma.file.findFirst({ where: { name: fullFileName } });
+    let existing =
+      reservedNames?.has(fullFileName) || (await prisma.file.findFirst({ where: { name: fullFileName } }));

    if (existing && (override || format === 'name')) {
      return { error: 'file with the same name already exists' };
@@ -101,9 +99,11 @@ export async function getFilename(
      if (!fileName) return { error: 'invalid file name' };

      fullFileName = `${fileName}${extension}`;
-      existing = await prisma.file.findFirst({ where: { name: fullFileName } });
+      existing =
+        reservedNames?.has(fullFileName) || (await prisma.file.findFirst({ where: { name: fullFileName } }));
    }

+    reservedNames?.add(fullFileName);
    return { fileName };
  } catch (e) {
    logger.warn(`error generating file name: ${e}`);
@@ -1,6 +1,5 @@
 import { prisma } from '@/lib/db';
-import { tmpdir } from 'os';
-import { join } from 'path';
+import { resolve } from 'path';

 export const DATABASE_TO_PROP = {
  coreReturnHttpsUrls: 'core.returnHttpsUrls',
@@ -154,7 +153,7 @@ export async function readDatabaseSettings() {
  if (!ziplineTable) {
    ziplineTable = await prisma.zipline.create({
      data: {
-        coreTempDirectory: join(tmpdir(), 'zipline'),
+        coreTempDirectory: resolve('./uploads/.tmp'),
      },
      omit: {
        createdAt: true,
@@ -1,5 +1,4 @@
-import { tmpdir } from 'os';
-import { join, resolve } from 'path';
+import { resolve } from 'path';
 import { z } from 'zod';
 import { log } from '../logger';
 import { ParsedConfig } from './read';
@@ -99,7 +98,7 @@ export const schema = z.object({
    tempDirectory: z
      .string()
      .transform((s) => resolve(s))
-      .default(join(tmpdir(), 'zipline')),
+      .default(resolve('./uploads/.tmp')),
    trustProxy: z.boolean().default(false),

    databaseUrl: z.url(),
@@ -101,15 +101,15 @@ export class LocalDatasource extends Datasource {
  }

  public async totalSize(): Promise<number> {
-    const files = await readdir(this.dir);
-    const sizes = await Promise.all(files.map((file) => this.size(file)));
+    const files = (await readdir(this.dir, { withFileTypes: true })).filter((file) => file.isFile());
+    const sizes = await Promise.all(files.map((file) => this.size(file.name)));

    return sizes.reduce((a, b) => a + b, 0);
  }

  public async clear(): Promise<void> {
-    for (const file of await readdir(this.dir)) {
-      await rm(join(this.dir, file));
+    for (const file of await readdir(this.dir, { withFileTypes: true })) {
+      if (file.isFile()) await rm(join(this.dir, file.name));
    }
  }

@@ -13,7 +13,9 @@ import {
  UploadPartCopyCommand,
 } from '@aws-sdk/client-s3';
 import { NodeHttpHandler } from '@smithy/node-http-handler';
+import { Upload } from '@aws-sdk/lib-storage';
 import { createReadStream } from 'fs';
+import { stat } from 'fs/promises';
 import { Agent as HttpAgent } from 'http';
 import { Agent as HttpsAgent } from 'https';
 import { Readable } from 'stream';
@@ -168,36 +170,49 @@ export class S3Datasource extends Datasource {
  }

  public async put(file: string, data: Buffer | string, options: PutOptions = {}): Promise<void> {
-    let command = new PutObjectCommand({
-      Bucket: this.options.bucket,
-      Key: this.key(file),
-      Body: data,
-      ...(options.mimetype ? { ContentType: options.mimetype } : {}),
-    });
+    try {
+      if (typeof data === 'string') {
+        const size = await stat(data).then((file) => file.size);
+        if (size > 25 * 1024 * 1024) {
+          // 25mb
+          this.logger.debug('putting object with multipart upload', { file, key: this.key(file) });

-    if (typeof data === 'string') {
-      const readStream = createReadStream(data);
-      command = new PutObjectCommand({
+          try {
+            const upload = new Upload({
+              client: this.client,
+              params: {
+                Bucket: this.options.bucket,
+                Key: this.key(file),
+                Body: createReadStream(data),
+                ...(options.mimetype ? { ContentType: options.mimetype } : {}),
+              },
+              leavePartsOnError: false,
+            });
+
+            await upload.done();
+            return;
+          } catch (error) {
+            this.logger.warn('multipart upload failed, retrying with a single request', {
+              error: error instanceof Error ? error.message : error,
+            });
+          }
+        }
+      }
+
+      const command = new PutObjectCommand({
        Bucket: this.options.bucket,
        Key: this.key(file),
-        Body: readStream,
+        Body: typeof data === 'string' ? createReadStream(data) : data,
        ...(options.mimetype ? { ContentType: options.mimetype } : {}),
      });
-
-      this.logger.debug('putting object from stream', { file, key: this.key(file) });
-    }
-
-    try {
      const res = await this.client.send(command);

      if (!isOk(res.$metadata.httpStatusCode || 0)) {
-        this.logger.error(
-          'there was an error while putting object',
-          res.$metadata as Record<string, unknown>,
-        );
+        throw new Error(`S3 put failed with status ${res.$metadata.httpStatusCode ?? 'unknown'}`);
      }
    } catch (e) {
      this.logger.error('there was an error while putting object', e as Record<string, unknown>);
+      throw e;
    }
  }

@@ -1,14 +1,12 @@
-import { tmpdir } from 'os';
 import { prisma } from '..';
-import { join } from 'path';
+import { resolve } from 'path';

 export async function getZipline() {
  const zipline = await prisma.zipline.findFirst();
  if (!zipline) {
-    const tmp = join(tmpdir(), 'zipline');
    return prisma.zipline.create({
      data: {
-        coreTempDirectory: tmp,
+        coreTempDirectory: resolve('./uploads/.tmp'),
      },
    });
  }
@@ -0,0 +1,27 @@
+export async function mapConcurrent<T, R>(
+  values: T[],
+  concurrency: number,
+  map: (value: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const results = Array<R>(values.length);
+  let next = 0;
+  let failed = false;
+  let failure: unknown;
+
+  async function worker() {
+    while (!failed && next < values.length) {
+      const index = next++;
+      try {
+        results[index] = await map(values[index], index);
+      } catch (error) {
+        failed = true;
+        failure = error;
+      }
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, worker));
+  if (failed) throw failure;
+
+  return results;
+}
@@ -2,13 +2,12 @@ import { readFile } from 'fs/promises';

 export type Mimes = [string, string[]][];

+const mimes = readFile(new URL('../../mimes.json', import.meta.url), 'utf8').then(
+  (data) => new Map((JSON.parse(data) as Mimes).map(([extension, types]) => [extension, types[0]])),
+);
+
 export async function guess(extension: string | null): Promise<string> {
  if (!extension) return 'application/octet-stream';

-  const mimes: Mimes = JSON.parse(await readFile('./mimes.json', 'utf8'));
-
-  const mime = mimes.find((x) => x[0] === extension);
-  if (!mime) return 'application/octet-stream';
-
-  return mime[1][0];
+  return (await mimes).get(extension) ?? 'application/octet-stream';
 }
@@ -1,15 +1,18 @@
 import { Config } from '../config/validate';
+import { log } from '../logger';
 import { onUpload as discordOnUpload, onShorten as discordOnShorten } from './discord';
 import { onUpload as httpOnUpload, onShorten as httpOnShorten } from './http';

-export async function onUpload(config: Config, args: Parameters<typeof discordOnUpload>[1]) {
-  Promise.all([discordOnUpload(config, args), httpOnUpload(config, args)]);
+const logger = log('webhooks');

-  return;
+export function onUpload(config: Config, args: Parameters<typeof discordOnUpload>[1]) {
+  void Promise.all([discordOnUpload(config, args), httpOnUpload(config, args)]).catch((error) =>
+    logger.error('upload webhook failed', { error: error instanceof Error ? error.message : error }),
+  );
 }

-export async function onShorten(config: Config, args: Parameters<typeof discordOnShorten>[1]) {
-  Promise.all([discordOnShorten(config, args), httpOnShorten(config, args)]);
-
-  return;
+export function onShorten(config: Config, args: Parameters<typeof discordOnShorten>[1]) {
+  void Promise.all([discordOnShorten(config, args), httpOnShorten(config, args)]).catch((error) =>
+    logger.error('shorten webhook failed', { error: error instanceof Error ? error.message : error }),
+  );
 }
@@ -4,7 +4,7 @@ import { getDatasource } from '@/lib/datasource';
 import { S3Datasource } from '@/lib/datasource/S3';
 import { File, fileSelect } from '@/lib/db/models/file';
 import { IncompleteFile } from '@/lib/db/models/incompleteFile';
-import { User, userSelect } from '@/lib/db/models/user';
+import { limitedUserSelect, User, userSelect } from '@/lib/db/models/user';
 import { log } from '@/lib/logger';
 import { randomCharacters } from '@/lib/random';
 import { UploadOptions } from '@/lib/uploader/parseHeaders';
@@ -192,7 +192,7 @@ async function runComplete(id: string, size: number) {
    where: {
      id: user.id,
    },
-    select: userSelect,
+    select: config.discord?.onUpload || config.httpWebhook.onUpload ? userSelect : limitedUserSelect,
  });
  if (!userr) return;

@@ -1,7 +1,7 @@
 import { config } from '@/lib/config';
 import { decryptToken } from '@/lib/crypto';
 import { prisma } from '@/lib/db';
-import { User, userSelect } from '@/lib/db/models/user';
+import { limitedUserSelect, User, userSelect } from '@/lib/db/models/user';
 import { FastifyReply } from 'fastify';
 import { FastifyRequest } from 'fastify/types/request';
 import { getSession } from '../session';
@@ -41,10 +41,14 @@ export function parseUserToken(
 }

 export async function userMiddleware(req: FastifyRequest, res: FastifyReply) {
+  const path = req.url.toLowerCase().split('?')[0];
+  const upload = ['/api/upload', '/api/upload/partial'].includes(path);
+  const leanUpload = upload && !config.discord?.onUpload && !config.httpWebhook.onUpload;
+
  // conditions met to allow anonymous folder uploads but later handled in the upload route
  const anonFolderUpload =
    req.headers['x-zipline-folder'] &&
-    ['/api/upload', '/api/upload/partial'].includes(req.url.toLowerCase().split('?')[0]) &&
+    upload &&
    !req.headers.authorization &&
    !req.cookies['zipline_session'];
  if (anonFolderUpload) return;
@@ -58,11 +62,11 @@ export async function userMiddleware(req: FastifyRequest, res: FastifyReply) {
      where: {
        token,
      },
-      select: userSelect,
+      select: leanUpload ? limitedUserSelect : userSelect,
    });
    if (!user) throw new ApiError(2001);

-    req.user = user;
+    req.user = user as User;

    return;
  }
@@ -80,9 +84,9 @@ export async function userMiddleware(req: FastifyRequest, res: FastifyReply) {
        },
      },
    },
-    select: userSelect,
+    select: leanUpload ? limitedUserSelect : userSelect,
  });
  if (!user) throw new ApiError(2001);

-  req.user = user;
+  req.user = user as User;
 }
@@ -11,6 +11,7 @@ import { userSelect } from '@/lib/db/models/user';
 import { sanitizeFilename } from '@/lib/fs';
 import { removeGps } from '@/lib/gps';
 import { log } from '@/lib/logger';
+import { mapConcurrent } from '@/lib/mapConcurrent';
 import { runThumbnailWorkers } from '@/lib/tasks/run/thumbnails';
 import { parseHeaders, UploadHeaders } from '@/lib/uploader/parseHeaders';
 import { onUpload } from '@/lib/webhooks';
@@ -18,7 +19,6 @@ import { Prisma } from '@/prisma/client';
 import { userMiddleware } from '@/server/middleware/user';
 import typedPlugin from '@/server/typedPlugin';
 import { SavedMultipartFile } from '@fastify/multipart';
-import { stat } from 'fs/promises';
 import { z } from 'zod';

 export type ApiUploadResponse = {
@@ -29,7 +29,7 @@ export type ApiUploadResponse = {
    url: string;
    pending?: boolean;
    removedGps?: boolean;
-    compressed?: CompressResult;
+    compressed?: Omit<CompressResult, 'buffer'>;
  }[];

  deletesAt?: string;
@@ -135,7 +135,7 @@ export default typedPlugin(
          ...(options.deletesAt && {
            deletesAt: options.deletesAt === 'never' ? 'never' : options.deletesAt.toISOString(),
          }),
-          ...(config.files.assumeMimetypes && { assumedMimetypes: Array(req.files.length) }),
+          ...(config.files.assumeMimetypes && { assumedMimetypes: Array(files.length) }),
        };

        const domain = getDomain(
@@ -146,8 +146,8 @@ export default typedPlugin(

        logger.debug('uploading files', { files: files.map((x) => x.filename) });

-        for (let i = 0; i !== files.length; ++i) {
-          const file = files[i];
+        // todo: maybe make configurable?
+        const prepared = await mapConcurrent(files, 4, async (file, i) => {
          const extension = getExtension(file.filename, options.overrides?.extension);

          if (config.files.disabledExtensions.includes(extension))
@@ -158,13 +158,6 @@ export default typedPlugin(
              `file[${i}]: File size is too large. Maximum file size is ${bytes(config.files.maxFileSize)} bytes`,
            );

-          // determine filename
-          const format = options.format || config.files.defaultFormat;
-          const nameResult = await getFilename(format, file.filename, extension, options.overrides?.filename);
-          if ('error' in nameResult) throw new ApiError(1009, `file[${i}]: ${nameResult.error}`);
-
-          const { fileName } = nameResult;
-
          // determine mimetype
          const { assumed, ...mimeRes } = await getMimetype(file.mimetype, extension);
          let mimetype = mimeRes.mimetype;
@@ -183,7 +176,6 @@ export default typedPlugin(
          }

          if (config.files.disabledTypes.includes(mimetype.trim().toLowerCase())) {
-            console.log(mimetype, config.files.disabledTypesDefault);
            if (config.files.disabledTypesDefault) mimetype = config.files.disabledTypesDefault;
            else throw new ApiError(1065, `file[${i}]: File type ${mimetype} is not allowed`);
          }
@@ -207,18 +199,46 @@ export default typedPlugin(
          // remove gps metadata if requested
          let removedGps = false;
          if (mimetype.startsWith('image/') && config.files.removeGpsMetadata) {
-            const removed = removeGps(file.filepath);
+            const removed = removeGps(compressed?.buffer ?? file.filepath);
            if (removed) logger.c('gps').debug(`removed gps metadata from ${file.filename}`);

            removedGps = removed;
          }

-          const tempFileStats = await stat(file.filepath);
+          return {
+            file,
+            extension: compressed ? `.${compressed.ext}` : extension,
+            mimetype: compressed?.mimetype ?? mimetype,
+            size: compressed?.buffer.length ?? file.file.bytesRead,
+            compressed,
+            removedGps,
+          };
+        });
+
+        const reservedNames = new Set<string>();
+        const format = options.format || config.files.defaultFormat;
+        const named: ((typeof prepared)[number] & { fileName: string })[] = [];
+        for (let i = 0; i < prepared.length; i++) {
+          const item = prepared[i];
+          const nameResult = await getFilename(
+            format,
+            item.file.filename,
+            item.extension,
+            options.overrides?.filename,
+            reservedNames,
+          );
+          if ('error' in nameResult) throw new ApiError(1009, `file[${i}]: ${nameResult.error}`);
+
+          named.push({ ...item, fileName: nameResult.fileName });
+        }
+
+        response.files = await mapConcurrent(named, 4, async (item, i) => {
+          const { file, fileName, extension, mimetype, size, compressed, removedGps } = item;

          const data: Prisma.FileCreateInput = {
-            name: `${fileName}${compressed ? '.' + compressed.ext : extension}`,
-            size: compressed?.buffer?.length ?? tempFileStats.size,
-            type: compressed?.mimetype ?? mimetype,
+            name: `${fileName}${extension}`,
+            size,
+            type: mimetype,
            User: { connect: { id: req.user ? req.user.id : options.folder ? folder?.userId : undefined } },
          };

@@ -241,24 +261,32 @@ export default typedPlugin(
            select: fileSelect,
          });

-          await datasource.put(fileUpload.name, compressed?.buffer ?? file.filepath, {
+          const storageData = compressed?.buffer ?? file.filepath;
+          await datasource.put(fileUpload.name, storageData, {
            mimetype: fileUpload.type,
          });
+          if (typeof storageData === 'string' && datasource.name === 'local' && req.tmpUploads) {
+            req.tmpUploads = req.tmpUploads.filter((path) => path !== storageData);
+          }

          const responseUrl = `${domain}${config.files.route === '/' || config.files.route === '' ? '' : `${config.files.route}`}/${fileUpload.name}`;

-          response.files.push({
+          const compressedResponse = compressed
+            ? { mimetype: compressed.mimetype, ext: compressed.ext, failed: compressed.failed }
+            : undefined;
+
+          const responseFile = {
            id: fileUpload.id,
            name: fileUpload.name,
            type: fileUpload.type,
            url: encodeURI(responseUrl),
            removedGps: removedGps || undefined,
-            compressed: compressed || undefined,
-          });
+            compressed: compressedResponse,
+          };

          logger.info(
            `${req.user ? req.user.username : '[anonymous folder upload]'} uploaded ${fileUpload.name}`,
-            { size: bytes(compressed?.buffer?.length ?? fileUpload.size), ip: req.ip },
+            { size: bytes(fileUpload.size), ip: req.ip },
          );

          await onUpload(config, {
@@ -275,7 +303,9 @@ export default typedPlugin(
              returned: encodeURI(responseUrl),
            },
          });
-        }
+
+          return responseFile;
+        });

        if (options.noJson)
          return res
@@ -4,7 +4,7 @@ import { bytes } from '@/lib/bytes';
 import { config } from '@/lib/config';
 import { hashPassword } from '@/lib/crypto';
 import { prisma } from '@/lib/db';
-import { userSelect } from '@/lib/db/models/user';
+import { limitedUserSelect } from '@/lib/db/models/user';
 import { sanitizeFilename } from '@/lib/fs';
 import { log } from '@/lib/logger';
 import { guess } from '@/lib/mimes';
@@ -99,7 +99,7 @@ export default typedPlugin(
          ...(options.deletesAt && {
            deletesAt: options.deletesAt === 'never' ? 'never' : options.deletesAt.toISOString(),
          }),
-          ...(config.files.assumeMimetypes && { assumedMimetypes: Array(req.files.length) }),
+          ...(config.files.assumeMimetypes && { assumedMimetypes: Array(files.length) }),
        };

        const domain = getDomain(
@@ -144,7 +144,7 @@ export default typedPlugin(
        const quotaUser = req.user
          ? req.user
          : folder?.userId
-            ? await prisma.user.findUnique({ where: { id: folder.userId }, select: userSelect })
+            ? await prisma.user.findUnique({ where: { id: folder.userId }, select: limitedUserSelect })
            : null;

        // check quota, using the current added length, and only just adding one file
@@ -175,6 +175,7 @@ export default typedPlugin(

        const tempFile = join(config.core.tempDirectory, sanitized);
        await rename(file.filepath, tempFile);
+        if (req.tmpUploads) req.tmpUploads = req.tmpUploads.filter((path) => path !== file.filepath);

        if (options.partial.lastchunk) {
          const extension = getExtension(options.partial.filename, options.overrides?.extension);