fix: export issues (#915 )

feat: log memory usage with DEBUG_MEMORY_LOG
2025-12-05 20:40:12 -08:00 · 2025-10-27 15:05:01 -07:00 · 2025-10-27 15:01:19 -07:00
5 changed files with 324 additions and 98 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -48,4 +48,5 @@ yarn-error.log*
 uploads*/
 *.crt
 *.key
-src/prisma
+src/prisma
+.memory.log.json
--- a/package.json
+++ b/package.json
@@ -49,6 +49,7 @@
    "@prisma/migrate": "6.13.0",
    "@smithy/node-http-handler": "^4.1.1",
    "@tabler/icons-react": "^3.34.1",
+    "archiver": "^7.0.1",
    "argon2": "^0.44.0",
    "asciinema-player": "^3.10.0",
    "bytes": "^3.1.2",
@@ -62,7 +63,6 @@
    "fast-glob": "^3.3.3",
    "fastify": "^5.5.0",
    "fastify-plugin": "^5.0.1",
-    "fflate": "^0.8.2",
    "fluent-ffmpeg": "^2.1.3",
    "highlight.js": "^11.11.1",
    "iron-session": "^8.0.4",
@@ -88,6 +88,7 @@
    "zustand": "^5.0.8"
  },
  "devDependencies": {
+    "@types/archiver": "^7.0.0",
    "@types/bytes": "^3.1.5",
    "@types/fluent-ffmpeg": "^2.1.27",
    "@types/katex": "^0.16.7",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -92,6 +92,9 @@ importers:
      '@tabler/icons-react':
        specifier: ^3.34.1
        version: 3.34.1(react@19.1.1)
+      archiver:
+        specifier: ^7.0.1
+        version: 7.0.1
      argon2:
        specifier: ^0.44.0
        version: 0.44.0
@@ -131,9 +134,6 @@ importers:
      fastify-plugin:
        specifier: ^5.0.1
        version: 5.0.1
-      fflate:
-        specifier: ^0.8.2
-        version: 0.8.2
      fluent-ffmpeg:
        specifier: ^2.1.3
        version: 2.1.3
@@ -204,6 +204,9 @@ importers:
        specifier: ^5.0.8
        version: 5.0.8(@types/react@19.1.12)(react@19.1.1)(use-sync-external-store@1.5.0(react@19.1.1))
    devDependencies:
+      '@types/archiver':
+        specifier: ^7.0.0
+        version: 7.0.0
      '@types/bytes':
        specifier: ^3.1.5
        version: 3.1.5
@@ -1954,6 +1957,9 @@ packages:
  '@tabler/icons@3.34.1':
    resolution: {integrity: sha512-9gTnUvd7Fd/DmQgr3MKY+oJLa1RfNsQo8c/ir3TJAWghOuZXodbtbVp0QBY2DxWuuvrSZFys0HEbv1CoiI5y6A==}

+  '@types/archiver@7.0.0':
+    resolution: {integrity: sha512-/3vwGwx9n+mCQdYZ2IKGGHEFL30I96UgBlk8EtRDDFQ9uxM1l4O5Ci6r00EMAkiDaTqD9DQ6nVrWRICnBPtzzg==}
+
  '@types/babel__core@7.20.5':
    resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==}

@@ -2070,6 +2076,9 @@ packages:
  '@types/react@19.1.12':
    resolution: {integrity: sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w==}

+  '@types/readdir-glob@1.1.5':
+    resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
+
  '@types/send@0.17.5':
    resolution: {integrity: sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w==}

@@ -2156,6 +2165,10 @@ packages:
    peerDependencies:
      vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0

+  abort-controller@3.0.0:
+    resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
+    engines: {node: '>=6.5'}
+
  abstract-logging@2.0.1:
    resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==}

@@ -2213,6 +2226,14 @@ packages:
  append-field@1.0.0:
    resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==}

+  archiver-utils@5.0.2:
+    resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==}
+    engines: {node: '>= 14'}
+
+  archiver@7.0.1:
+    resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==}
+    engines: {node: '>= 14'}
+
  arg@5.0.2:
    resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==}

@@ -2272,6 +2293,9 @@ packages:
  async@0.2.10:
    resolution: {integrity: sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==}

+  async@3.2.6:
+    resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==}
+
  atomic-sleep@1.0.0:
    resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
    engines: {node: '>=8.0.0'}
@@ -2295,12 +2319,28 @@ packages:
    resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==}
    engines: {node: '>= 0.4'}

+  b4a@1.7.3:
+    resolution: {integrity: sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==}
+    peerDependencies:
+      react-native-b4a: '*'
+    peerDependenciesMeta:
+      react-native-b4a:
+        optional: true
+
  bail@2.0.2:
    resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==}

  balanced-match@1.0.2:
    resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}

+  bare-events@2.8.1:
+    resolution: {integrity: sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==}
+    peerDependencies:
+      bare-abort-controller: '*'
+    peerDependenciesMeta:
+      bare-abort-controller:
+        optional: true
+
  base64-js@1.5.1:
    resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}

@@ -2326,12 +2366,19 @@ packages:
    engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}
    hasBin: true

+  buffer-crc32@1.0.0:
+    resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==}
+    engines: {node: '>=8.0.0'}
+
  buffer-from@1.1.2:
    resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}

  buffer@5.6.0:
    resolution: {integrity: sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==}

+  buffer@6.0.3:
+    resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
+
  bundle-require@5.1.0:
    resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==}
    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -2462,6 +2509,10 @@ packages:
    resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==}
    engines: {node: ^12.20.0 || >=14}

+  compress-commons@6.0.2:
+    resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==}
+    engines: {node: '>= 14'}
+
  concat-map@0.0.1:
    resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}

@@ -2494,6 +2545,18 @@ packages:
    resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==}
    engines: {node: '>=18'}

+  core-util-is@1.0.3:
+    resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
+
+  crc-32@1.2.2:
+    resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==}
+    engines: {node: '>=0.8'}
+    hasBin: true
+
+  crc32-stream@6.0.0:
+    resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==}
+    engines: {node: '>= 14'}
+
  cross-env@10.0.0:
    resolution: {integrity: sha512-aU8qlEK/nHYtVuN4p7UQgAwVljzMg8hB4YK5ThRqD2l/ziSnryncPNn7bMLt5cFYsKVKBh8HqLqyCoTupEUu7Q==}
    engines: {node: '>=20'}
@@ -2853,9 +2916,16 @@ packages:
    resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==}
    engines: {node: '>=0.10.0'}

+  event-target-shim@5.0.1:
+    resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
+    engines: {node: '>=6'}
+
  eventemitter3@4.0.7:
    resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}

+  events-universal@1.0.1:
+    resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==}
+
  events@3.3.0:
    resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
    engines: {node: '>=0.8.x'}
@@ -2883,6 +2953,9 @@ packages:
    resolution: {integrity: sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==}
    engines: {node: '>=6.0.0'}

+  fast-fifo@1.3.2:
+    resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==}
+
  fast-glob@3.3.3:
    resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==}
    engines: {node: '>=8.6.0'}
@@ -2936,9 +3009,6 @@ packages:
      picomatch:
        optional: true

-  fflate@0.8.2:
-    resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
-
  file-entry-cache@8.0.0:
    resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==}
    engines: {node: '>=16.0.0'}
@@ -3314,6 +3384,10 @@ packages:
    resolution: {integrity: sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==}
    engines: {node: '>= 0.4'}

+  is-stream@2.0.1:
+    resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
+    engines: {node: '>=8'}
+
  is-string@1.1.1:
    resolution: {integrity: sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==}
    engines: {node: '>= 0.4'}
@@ -3338,6 +3412,9 @@ packages:
    resolution: {integrity: sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==}
    engines: {node: '>= 0.4'}

+  isarray@1.0.0:
+    resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==}
+
  isarray@2.0.5:
    resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==}

@@ -3437,6 +3514,10 @@ packages:
    resolution: {integrity: sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==}
    engines: {node: '>=0.10'}

+  lazystream@1.0.1:
+    resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==}
+    engines: {node: '>= 0.6.3'}
+
  levn@0.4.1:
    resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}
    engines: {node: '>= 0.8.0'}
@@ -3676,6 +3757,10 @@ packages:
  minimatch@3.1.2:
    resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==}

+  minimatch@5.1.6:
+    resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==}
+    engines: {node: '>=10'}
+
  minimatch@9.0.5:
    resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==}
    engines: {node: '>=16 || 14 >=14.17'}
@@ -4056,12 +4141,19 @@ packages:
      typescript:
        optional: true

+  process-nextick-args@2.0.1:
+    resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}
+
  process-warning@4.0.1:
    resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==}

  process-warning@5.0.0:
    resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==}

+  process@0.11.10:
+    resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
+    engines: {node: '>= 0.6.0'}
+
  prompts@2.4.2:
    resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
    engines: {node: '>= 6'}
@@ -4214,10 +4306,20 @@ packages:
    resolution: {integrity: sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==}
    engines: {node: '>=18'}

+  readable-stream@2.3.8:
+    resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
+
  readable-stream@3.6.2:
    resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
    engines: {node: '>= 6'}

+  readable-stream@4.7.0:
+    resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==}
+    engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
+
+  readdir-glob@1.1.3:
+    resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==}
+
  readdirp@3.6.0:
    resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
    engines: {node: '>=8.10.0'}
@@ -4317,6 +4419,9 @@ packages:
    resolution: {integrity: sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==}
    engines: {node: '>=0.4'}

+  safe-buffer@5.1.2:
+    resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==}
+
  safe-buffer@5.2.1:
    resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}

@@ -4486,6 +4591,9 @@ packages:
    resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
    engines: {node: '>=10.0.0'}

+  streamx@2.23.0:
+    resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==}
+
  string-width@4.2.3:
    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
    engines: {node: '>=8'}
@@ -4517,6 +4625,9 @@ packages:
    resolution: {integrity: sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==}
    engines: {node: '>= 0.4'}

+  string_decoder@1.1.1:
+    resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==}
+
  string_decoder@1.3.0:
    resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}

@@ -4578,6 +4689,12 @@ packages:
  tabbable@6.2.0:
    resolution: {integrity: sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==}

+  tar-stream@3.1.7:
+    resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==}
+
+  text-decoder@1.2.3:
+    resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==}
+
  thenify-all@1.6.0:
    resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==}
    engines: {node: '>=0.8'}
@@ -5006,6 +5123,10 @@ packages:
    resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
    engines: {node: '>=10'}

+  zip-stream@6.0.1:
+    resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==}
+    engines: {node: '>= 14'}
+
  zod@4.1.5:
    resolution: {integrity: sha512-rcUUZqlLJgBC33IT3PNMgsCq6TzLQEG/Ei/KTCU0PedSWRMAXoOUN+4t/0H+Q8bdnLPdqUYnvboJT0bn/229qg==}

@@ -7046,6 +7167,10 @@ snapshots:

  '@tabler/icons@3.34.1': {}

+  '@types/archiver@7.0.0':
+    dependencies:
+      '@types/readdir-glob': 1.1.5
+
  '@types/babel__core@7.20.5':
    dependencies:
      '@babel/parser': 7.28.3
@@ -7177,6 +7302,10 @@ snapshots:
    dependencies:
      csstype: 3.1.3

+  '@types/readdir-glob@1.1.5':
+    dependencies:
+      '@types/node': 24.3.0
+
  '@types/send@0.17.5':
    dependencies:
      '@types/mime': 1.3.5
@@ -7304,6 +7433,10 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

+  abort-controller@3.0.0:
+    dependencies:
+      event-target-shim: 5.0.1
+
  abstract-logging@2.0.1: {}

  acorn-jsx@5.3.2(acorn@8.15.0):
@@ -7351,6 +7484,29 @@ snapshots:

  append-field@1.0.0: {}

+  archiver-utils@5.0.2:
+    dependencies:
+      glob: 10.4.5
+      graceful-fs: 4.2.11
+      is-stream: 2.0.1
+      lazystream: 1.0.1
+      lodash: 4.17.21
+      normalize-path: 3.0.0
+      readable-stream: 4.7.0
+
+  archiver@7.0.1:
+    dependencies:
+      archiver-utils: 5.0.2
+      async: 3.2.6
+      buffer-crc32: 1.0.0
+      readable-stream: 4.7.0
+      readdir-glob: 1.1.3
+      tar-stream: 3.1.7
+      zip-stream: 6.0.1
+    transitivePeerDependencies:
+      - bare-abort-controller
+      - react-native-b4a
+
  arg@5.0.2: {}

  argon2@0.44.0:
@@ -7434,6 +7590,8 @@ snapshots:

  async@0.2.10: {}

+  async@3.2.6: {}
+
  atomic-sleep@1.0.0: {}

  attr-accept@2.2.5: {}
@@ -7451,10 +7609,14 @@ snapshots:

  axobject-query@4.1.0: {}

+  b4a@1.7.3: {}
+
  bail@2.0.2: {}

  balanced-match@1.0.2: {}

+  bare-events@2.8.1: {}
+
  base64-js@1.5.1: {}

  binary-extensions@2.3.0: {}
@@ -7481,6 +7643,8 @@ snapshots:
      node-releases: 2.0.19
      update-browserslist-db: 1.1.3(browserslist@4.25.4)

+  buffer-crc32@1.0.0: {}
+
  buffer-from@1.1.2: {}

  buffer@5.6.0:
@@ -7488,6 +7652,11 @@ snapshots:
      base64-js: 1.5.1
      ieee754: 1.2.1

+  buffer@6.0.3:
+    dependencies:
+      base64-js: 1.5.1
+      ieee754: 1.2.1
+
  bundle-require@5.1.0(esbuild@0.25.5):
    dependencies:
      esbuild: 0.25.5
@@ -7614,6 +7783,14 @@ snapshots:

  commander@9.5.0: {}

+  compress-commons@6.0.2:
+    dependencies:
+      crc-32: 1.2.2
+      crc32-stream: 6.0.0
+      is-stream: 2.0.1
+      normalize-path: 3.0.0
+      readable-stream: 4.7.0
+
  concat-map@0.0.1: {}

  concat-stream@2.0.0:
@@ -7639,6 +7816,15 @@ snapshots:

  cookie@1.0.2: {}

+  core-util-is@1.0.3: {}
+
+  crc-32@1.2.2: {}
+
+  crc32-stream@6.0.0:
+    dependencies:
+      crc-32: 1.2.2
+      readable-stream: 4.7.0
+
  cross-env@10.0.0:
    dependencies:
      '@epic-web/invariant': 1.0.0
@@ -8124,8 +8310,16 @@ snapshots:

  esutils@2.0.3: {}

+  event-target-shim@5.0.1: {}
+
  eventemitter3@4.0.7: {}

+  events-universal@1.0.1:
+    dependencies:
+      bare-events: 2.8.1
+    transitivePeerDependencies:
+      - bare-abort-controller
+
  events@3.3.0: {}

  exsolve@1.0.7: {}
@@ -8144,6 +8338,8 @@ snapshots:

  fast-equals@5.2.2: {}

+  fast-fifo@1.3.2: {}
+
  fast-glob@3.3.3:
    dependencies:
      '@nodelib/fs.stat': 2.0.5
@@ -8209,8 +8405,6 @@ snapshots:
    optionalDependencies:
      picomatch: 4.0.3

-  fflate@0.8.2: {}
-
  file-entry-cache@8.0.0:
    dependencies:
      flat-cache: 4.0.1
@@ -8620,6 +8814,8 @@ snapshots:
    dependencies:
      call-bound: 1.0.4

+  is-stream@2.0.1: {}
+
  is-string@1.1.1:
    dependencies:
      call-bound: 1.0.4
@@ -8646,6 +8842,8 @@ snapshots:
      call-bound: 1.0.4
      get-intrinsic: 1.3.0

+  isarray@1.0.0: {}
+
  isarray@2.0.5: {}

  isexe@2.0.0: {}
@@ -8763,6 +8961,10 @@ snapshots:
    dependencies:
      language-subtag-registry: 0.3.23

+  lazystream@1.0.1:
+    dependencies:
+      readable-stream: 2.3.8
+
  levn@0.4.1:
    dependencies:
      prelude-ls: 1.2.1
@@ -9200,6 +9402,10 @@ snapshots:
    dependencies:
      brace-expansion: 1.1.12

+  minimatch@5.1.6:
+    dependencies:
+      brace-expansion: 2.0.2
+
  minimatch@9.0.5:
    dependencies:
      brace-expansion: 2.0.2
@@ -9572,10 +9778,14 @@ snapshots:
    transitivePeerDependencies:
      - magicast

+  process-nextick-args@2.0.1: {}
+
  process-warning@4.0.1: {}

  process-warning@5.0.0: {}

+  process@0.11.10: {}
+
  prompts@2.4.2:
    dependencies:
      kleur: 3.0.3
@@ -9741,12 +9951,34 @@ snapshots:
      type-fest: 4.41.0
      unicorn-magic: 0.1.0

+  readable-stream@2.3.8:
+    dependencies:
+      core-util-is: 1.0.3
+      inherits: 2.0.4
+      isarray: 1.0.0
+      process-nextick-args: 2.0.1
+      safe-buffer: 5.1.2
+      string_decoder: 1.1.1
+      util-deprecate: 1.0.2
+
  readable-stream@3.6.2:
    dependencies:
      inherits: 2.0.4
      string_decoder: 1.3.0
      util-deprecate: 1.0.2

+  readable-stream@4.7.0:
+    dependencies:
+      abort-controller: 3.0.0
+      buffer: 6.0.3
+      events: 3.3.0
+      process: 0.11.10
+      string_decoder: 1.3.0
+
+  readdir-glob@1.1.3:
+    dependencies:
+      minimatch: 5.1.6
+
  readdirp@3.6.0:
    dependencies:
      picomatch: 2.3.1
@@ -9917,6 +10149,8 @@ snapshots:
      has-symbols: 1.1.0
      isarray: 2.0.5

+  safe-buffer@5.1.2: {}
+
  safe-buffer@5.2.1: {}

  safe-push-apply@1.0.0:
@@ -10115,6 +10349,15 @@ snapshots:

  streamsearch@1.1.0: {}

+  streamx@2.23.0:
+    dependencies:
+      events-universal: 1.0.1
+      fast-fifo: 1.3.2
+      text-decoder: 1.2.3
+    transitivePeerDependencies:
+      - bare-abort-controller
+      - react-native-b4a
+
  string-width@4.2.3:
    dependencies:
      emoji-regex: 8.0.0
@@ -10177,6 +10420,10 @@ snapshots:
      define-properties: 1.2.1
      es-object-atoms: 1.1.1

+  string_decoder@1.1.1:
+    dependencies:
+      safe-buffer: 5.1.2
+
  string_decoder@1.3.0:
    dependencies:
      safe-buffer: 5.2.1
@@ -10240,6 +10487,21 @@ snapshots:

  tabbable@6.2.0: {}

+  tar-stream@3.1.7:
+    dependencies:
+      b4a: 1.7.3
+      fast-fifo: 1.3.2
+      streamx: 2.23.0
+    transitivePeerDependencies:
+      - bare-abort-controller
+      - react-native-b4a
+
+  text-decoder@1.2.3:
+    dependencies:
+      b4a: 1.7.3
+    transitivePeerDependencies:
+      - react-native-b4a
+
  thenify-all@1.6.0:
    dependencies:
      thenify: 3.3.1
@@ -10692,6 +10954,12 @@ snapshots:

  yocto-queue@0.1.0: {}

+  zip-stream@6.0.1:
+    dependencies:
+      archiver-utils: 5.0.2
+      compress-commons: 6.0.2
+      readable-stream: 4.7.0
+
  zod@4.1.5: {}

  zustand@5.0.8(@types/react@19.1.12)(react@19.1.1)(use-sync-external-store@1.5.0(react@19.1.1)):
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -20,7 +20,7 @@ import { fastifyRateLimit } from '@fastify/rate-limit';
 import { fastifySensible } from '@fastify/sensible';
 import { fastifyStatic } from '@fastify/static';
 import fastify from 'fastify';
-import { mkdir, readFile } from 'fs/promises';
+import { appendFile, mkdir, readFile, writeFile } from 'fs/promises';
 import ms, { StringValue } from 'ms';
 import { version } from '../../package.json';
 import { checkRateLimit } from './plugins/checkRateLimit';
@@ -284,6 +284,24 @@ async function main() {
  }

  tasks.start();
+
+  if (process.env.DEBUG_MONITOR_MEMORY === 'true') {
+    await writeFile('.memory.log.json', '', 'utf8');
+    setInterval(async () => {
+      const mu = process.memoryUsage();
+      const cpu = process.cpuUsage();
+
+      const entry = {
+        timestamp: new Date().toISOString(),
+        data: {
+          memoryUsage: mu,
+          cpuUsage: cpu,
+        },
+      };
+
+      await appendFile('.memory.log.json', JSON.stringify(entry) + '\n', 'utf8');
+    }, 1000);
+  }
 }

 main();
--- a/src/server/routes/api/user/export.ts
+++ b/src/server/routes/api/user/export.ts
@@ -5,11 +5,12 @@ import { log } from '@/lib/logger';
 import { secondlyRatelimit } from '@/lib/ratelimits';
 import { userMiddleware } from '@/server/middleware/user';
 import fastifyPlugin from 'fastify-plugin';
-import { Zip, ZipPassThrough } from 'fflate';
+import archiver from 'archiver';
 import { createWriteStream } from 'fs';
 import { rm, stat } from 'fs/promises';
 import { join } from 'path';
 import { Export } from '@/prisma/client';
+import { bytes } from '@/lib/bytes';

 export type ApiUserExportResponse = {
  running?: boolean;
@@ -92,70 +93,29 @@ export default fastifyPlugin(
          size: '0',
        },
      });
-
      const writeStream = createWriteStream(exportPath);
-      const zip = new Zip();

-      const onBackpressure = (stream: any, outputStream: any, cb: any) => {
-        const runCb = () => {
-          cb(applyOutputBackpressure || backpressureBytes > backpressureThreshold);
-        };
+      const zip = archiver('zip', {
+        zlib: { level: 9 },
+      });

-        const backpressureThreshold = 65536;
-        const backpressure: number[] = [];
-        let backpressureBytes = 0;
-        const push = stream.push;
-        stream.push = (data: string | any[], final: any) => {
-          backpressure.push(data.length);
-          backpressureBytes += data.length;
-          runCb();
-          push.call(stream, data, final);
-        };
-        let ondata = stream.ondata;
-        const ondataPatched = (err: any, data: any, final: any) => {
-          ondata.call(stream, err, data, final);
-          backpressureBytes -= backpressure.shift()!;
-          runCb();
-        };
-        if (ondata) {
-          stream.ondata = ondataPatched;
-        } else {
-          Object.defineProperty(stream, 'ondata', {
-            get: () => ondataPatched,
-            set: (cb) => (ondata = cb),
-          });
+      zip.pipe(writeStream);
+
+      let totalSize = 0;
+      for (const file of files) {
+        const stream = await datasource.get(file.name);
+        if (!stream) {
+          logger.warn(`failed to get file ${file.name}`);
+          continue;
        }

-        let applyOutputBackpressure = false;
-        const write = outputStream.write;
-        outputStream.write = (data: any) => {
-          const outputNotFull = write.call(outputStream, data);
-          applyOutputBackpressure = !outputNotFull;
-          runCb();
-        };
-        outputStream.on('drain', () => {
-          applyOutputBackpressure = false;
-          runCb();
-        });
-      };
+        zip.append(stream, { name: file.name });
+        totalSize += file.size;
+        logger.debug('file added to zip', { name: file.name, size: file.size });
+      }

-      zip.ondata = async (err, data, final) => {
-        if (err) {
-          writeStream.close();
-          logger.debug('error while writing to zip', { err });
-          logger.error(`export for ${req.user.id} failed`);
-
-          await prisma.export.delete({ where: { id: exportDb.id } });
-
-          return;
-        }
-
-        writeStream.write(data);
-
-        if (!final) return;
-
-        writeStream.end();
-        logger.debug('exported', { path: exportPath, bytes: data.length });
+      writeStream.on('close', async () => {
+        logger.debug('exported', { path: exportPath, bytes: zip.pointer() });
        logger.info(`export for ${req.user.id} finished at ${exportPath}`);

        await prisma.export.update({
@@ -165,37 +125,15 @@ export default fastifyPlugin(
            size: (await stat(exportPath)).size.toString(),
          },
        });
-      };
+      });

-      for (let i = 0; i !== files.length; ++i) {
-        const file = files[i];
+      zip.on('error', (err) => {
+        logger.error('export zip error', { err, exportId: exportDb.id });
+      });

-        const stream = await datasource.get(file.name);
-        if (!stream) {
-          logger.warn(`failed to get file ${file.name}`);
-          continue;
-        }
+      zip.finalize();

-        const passThrough = new ZipPassThrough(file.name);
-        zip.add(passThrough);
-
-        onBackpressure(passThrough, stream, (applyBackpressure: boolean) => {
-          if (applyBackpressure) {
-            stream.pause();
-          } else if (stream.isPaused()) {
-            stream.resume();
-          }
-        });
-        stream.on('data', (c) => passThrough.push(c));
-        stream.on('end', () => {
-          passThrough.push(new Uint8Array(0), true);
-          logger.debug(`file ${i + 1}/${files.length} added to zip`, { name: file.name });
-        });
-      }
-
-      zip.end();
-
-      logger.info(`export for ${req.user.id} started`);
+      logger.info(`export for ${req.user.id} started`, { totalSize: bytes(totalSize) });

      return res.send({ running: true });
    });
Author	SHA1	Message	Date
diced	ffbad41994	fix: export issues (#915 )	2025-10-27 15:05:01 -07:00
diced	2a6f1f418a	feat: log memory usage with DEBUG_MEMORY_LOG	2025-10-27 15:01:19 -07:00