From ffbad41994e02bd70ba217c205127b6be904dcab Mon Sep 17 00:00:00 2001 From: diced Date: Mon, 27 Oct 2025 15:05:01 -0700 Subject: [PATCH] fix: export issues (#915) --- package.json | 3 +- pnpm-lock.yaml | 284 ++++++++++++++++++++++++++- src/server/routes/api/user/export.ts | 112 +++-------- 3 files changed, 303 insertions(+), 96 deletions(-) diff --git a/package.json b/package.json index 726692f5..d179b78f 100755 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@prisma/migrate": "6.13.0", "@smithy/node-http-handler": "^4.1.1", "@tabler/icons-react": "^3.34.1", + "archiver": "^7.0.1", "argon2": "^0.44.0", "asciinema-player": "^3.10.0", "bytes": "^3.1.2", @@ -62,7 +63,6 @@ "fast-glob": "^3.3.3", "fastify": "^5.5.0", "fastify-plugin": "^5.0.1", - "fflate": "^0.8.2", "fluent-ffmpeg": "^2.1.3", "highlight.js": "^11.11.1", "iron-session": "^8.0.4", @@ -88,6 +88,7 @@ "zustand": "^5.0.8" }, "devDependencies": { + "@types/archiver": "^7.0.0", "@types/bytes": "^3.1.5", "@types/fluent-ffmpeg": "^2.1.27", "@types/katex": "^0.16.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6c92e55..1dda154c 100755 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -92,6 +92,9 @@ importers: '@tabler/icons-react': specifier: ^3.34.1 version: 3.34.1(react@19.1.1) + archiver: + specifier: ^7.0.1 + version: 7.0.1 argon2: specifier: ^0.44.0 version: 0.44.0 @@ -131,9 +134,6 @@ importers: fastify-plugin: specifier: ^5.0.1 version: 5.0.1 - fflate: - specifier: ^0.8.2 - version: 0.8.2 fluent-ffmpeg: specifier: ^2.1.3 version: 2.1.3 @@ -204,6 +204,9 @@ importers: specifier: ^5.0.8 version: 5.0.8(@types/react@19.1.12)(react@19.1.1)(use-sync-external-store@1.5.0(react@19.1.1)) devDependencies: + '@types/archiver': + specifier: ^7.0.0 + version: 7.0.0 '@types/bytes': specifier: ^3.1.5 version: 3.1.5 @@ -1954,6 +1957,9 @@ packages: '@tabler/icons@3.34.1': resolution: {integrity: sha512-9gTnUvd7Fd/DmQgr3MKY+oJLa1RfNsQo8c/ir3TJAWghOuZXodbtbVp0QBY2DxWuuvrSZFys0HEbv1CoiI5y6A==} + '@types/archiver@7.0.0': + resolution: {integrity: sha512-/3vwGwx9n+mCQdYZ2IKGGHEFL30I96UgBlk8EtRDDFQ9uxM1l4O5Ci6r00EMAkiDaTqD9DQ6nVrWRICnBPtzzg==} + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -2070,6 +2076,9 @@ packages: '@types/react@19.1.12': resolution: {integrity: sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w==} + '@types/readdir-glob@1.1.5': + resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==} + '@types/send@0.17.5': resolution: {integrity: sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w==} @@ -2156,6 +2165,10 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} @@ -2213,6 +2226,14 @@ packages: append-field@1.0.0: resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} @@ -2272,6 +2293,9 @@ packages: async@0.2.10: resolution: {integrity: sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==} + async@3.2.6: + resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} + atomic-sleep@1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} @@ -2295,12 +2319,28 @@ packages: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} + b4a@1.7.3: + resolution: {integrity: sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + bare-events@2.8.1: + resolution: {integrity: sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -2326,12 +2366,19 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} buffer@5.6.0: resolution: {integrity: sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==} + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bundle-require@5.1.0: resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2462,6 +2509,10 @@ packages: resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==} engines: {node: ^12.20.0 || >=14} + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -2494,6 +2545,18 @@ packages: resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==} engines: {node: '>=18'} + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + cross-env@10.0.0: resolution: {integrity: sha512-aU8qlEK/nHYtVuN4p7UQgAwVljzMg8hB4YK5ThRqD2l/ziSnryncPNn7bMLt5cFYsKVKBh8HqLqyCoTupEUu7Q==} engines: {node: '>=20'} @@ -2853,9 +2916,16 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -2883,6 +2953,9 @@ packages: resolution: {integrity: sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==} engines: {node: '>=6.0.0'} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -2936,9 +3009,6 @@ packages: picomatch: optional: true - fflate@0.8.2: - resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} - file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -3314,6 +3384,10 @@ packages: resolution: {integrity: sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==} engines: {node: '>= 0.4'} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-string@1.1.1: resolution: {integrity: sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==} engines: {node: '>= 0.4'} @@ -3338,6 +3412,9 @@ packages: resolution: {integrity: sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==} engines: {node: '>= 0.4'} + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + isarray@2.0.5: resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} @@ -3437,6 +3514,10 @@ packages: resolution: {integrity: sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==} engines: {node: '>=0.10'} + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -3676,6 +3757,10 @@ packages: minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + minimatch@5.1.6: + resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} + engines: {node: '>=10'} + minimatch@9.0.5: resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} @@ -4056,12 +4141,19 @@ packages: typescript: optional: true + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + process-warning@4.0.1: resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} process-warning@5.0.0: resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + prompts@2.4.2: resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} engines: {node: '>= 6'} @@ -4214,10 +4306,20 @@ packages: resolution: {integrity: sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==} engines: {node: '>=18'} + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -4317,6 +4419,9 @@ packages: resolution: {integrity: sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==} engines: {node: '>=0.4'} + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -4486,6 +4591,9 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} + streamx@2.23.0: + resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -4517,6 +4625,9 @@ packages: resolution: {integrity: sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==} engines: {node: '>= 0.4'} + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -4578,6 +4689,12 @@ packages: tabbable@6.2.0: resolution: {integrity: sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + + text-decoder@1.2.3: + resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -5006,6 +5123,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + zod@4.1.5: resolution: {integrity: sha512-rcUUZqlLJgBC33IT3PNMgsCq6TzLQEG/Ei/KTCU0PedSWRMAXoOUN+4t/0H+Q8bdnLPdqUYnvboJT0bn/229qg==} @@ -7046,6 +7167,10 @@ snapshots: '@tabler/icons@3.34.1': {} + '@types/archiver@7.0.0': + dependencies: + '@types/readdir-glob': 1.1.5 + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.28.3 @@ -7177,6 +7302,10 @@ snapshots: dependencies: csstype: 3.1.3 + '@types/readdir-glob@1.1.5': + dependencies: + '@types/node': 24.3.0 + '@types/send@0.17.5': dependencies: '@types/mime': 1.3.5 @@ -7304,6 +7433,10 @@ snapshots: transitivePeerDependencies: - supports-color + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + abstract-logging@2.0.1: {} acorn-jsx@5.3.2(acorn@8.15.0): @@ -7351,6 +7484,29 @@ snapshots: append-field@1.0.0: {} + archiver-utils@5.0.2: + dependencies: + glob: 10.4.5 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.21 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + arg@5.0.2: {} argon2@0.44.0: @@ -7434,6 +7590,8 @@ snapshots: async@0.2.10: {} + async@3.2.6: {} + atomic-sleep@1.0.0: {} attr-accept@2.2.5: {} @@ -7451,10 +7609,14 @@ snapshots: axobject-query@4.1.0: {} + b4a@1.7.3: {} + bail@2.0.2: {} balanced-match@1.0.2: {} + bare-events@2.8.1: {} + base64-js@1.5.1: {} binary-extensions@2.3.0: {} @@ -7481,6 +7643,8 @@ snapshots: node-releases: 2.0.19 update-browserslist-db: 1.1.3(browserslist@4.25.4) + buffer-crc32@1.0.0: {} + buffer-from@1.1.2: {} buffer@5.6.0: @@ -7488,6 +7652,11 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bundle-require@5.1.0(esbuild@0.25.5): dependencies: esbuild: 0.25.5 @@ -7614,6 +7783,14 @@ snapshots: commander@9.5.0: {} + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + concat-map@0.0.1: {} concat-stream@2.0.0: @@ -7639,6 +7816,15 @@ snapshots: cookie@1.0.2: {} + core-util-is@1.0.3: {} + + crc-32@1.2.2: {} + + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + cross-env@10.0.0: dependencies: '@epic-web/invariant': 1.0.0 @@ -8124,8 +8310,16 @@ snapshots: esutils@2.0.3: {} + event-target-shim@5.0.1: {} + eventemitter3@4.0.7: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.8.1 + transitivePeerDependencies: + - bare-abort-controller + events@3.3.0: {} exsolve@1.0.7: {} @@ -8144,6 +8338,8 @@ snapshots: fast-equals@5.2.2: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -8209,8 +8405,6 @@ snapshots: optionalDependencies: picomatch: 4.0.3 - fflate@0.8.2: {} - file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 @@ -8620,6 +8814,8 @@ snapshots: dependencies: call-bound: 1.0.4 + is-stream@2.0.1: {} + is-string@1.1.1: dependencies: call-bound: 1.0.4 @@ -8646,6 +8842,8 @@ snapshots: call-bound: 1.0.4 get-intrinsic: 1.3.0 + isarray@1.0.0: {} + isarray@2.0.5: {} isexe@2.0.0: {} @@ -8763,6 +8961,10 @@ snapshots: dependencies: language-subtag-registry: 0.3.23 + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -9200,6 +9402,10 @@ snapshots: dependencies: brace-expansion: 1.1.12 + minimatch@5.1.6: + dependencies: + brace-expansion: 2.0.2 + minimatch@9.0.5: dependencies: brace-expansion: 2.0.2 @@ -9572,10 +9778,14 @@ snapshots: transitivePeerDependencies: - magicast + process-nextick-args@2.0.1: {} + process-warning@4.0.1: {} process-warning@5.0.0: {} + process@0.11.10: {} + prompts@2.4.2: dependencies: kleur: 3.0.3 @@ -9741,12 +9951,34 @@ snapshots: type-fest: 4.41.0 unicorn-magic: 0.1.0 + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + readable-stream@3.6.2: dependencies: inherits: 2.0.4 string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.6 + readdirp@3.6.0: dependencies: picomatch: 2.3.1 @@ -9917,6 +10149,8 @@ snapshots: has-symbols: 1.1.0 isarray: 2.0.5 + safe-buffer@5.1.2: {} + safe-buffer@5.2.1: {} safe-push-apply@1.0.0: @@ -10115,6 +10349,15 @@ snapshots: streamsearch@1.1.0: {} + streamx@2.23.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.3 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -10177,6 +10420,10 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 @@ -10240,6 +10487,21 @@ snapshots: tabbable@6.2.0: {} + tar-stream@3.1.7: + dependencies: + b4a: 1.7.3 + fast-fifo: 1.3.2 + streamx: 2.23.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.3: + dependencies: + b4a: 1.7.3 + transitivePeerDependencies: + - react-native-b4a + thenify-all@1.6.0: dependencies: thenify: 3.3.1 @@ -10692,6 +10954,12 @@ snapshots: yocto-queue@0.1.0: {} + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + zod@4.1.5: {} zustand@5.0.8(@types/react@19.1.12)(react@19.1.1)(use-sync-external-store@1.5.0(react@19.1.1)): diff --git a/src/server/routes/api/user/export.ts b/src/server/routes/api/user/export.ts index fb05c0d5..1f6f143c 100644 --- a/src/server/routes/api/user/export.ts +++ b/src/server/routes/api/user/export.ts @@ -5,11 +5,12 @@ import { log } from '@/lib/logger'; import { secondlyRatelimit } from '@/lib/ratelimits'; import { userMiddleware } from '@/server/middleware/user'; import fastifyPlugin from 'fastify-plugin'; -import { Zip, ZipPassThrough } from 'fflate'; +import archiver from 'archiver'; import { createWriteStream } from 'fs'; import { rm, stat } from 'fs/promises'; import { join } from 'path'; import { Export } from '@/prisma/client'; +import { bytes } from '@/lib/bytes'; export type ApiUserExportResponse = { running?: boolean; @@ -92,70 +93,29 @@ export default fastifyPlugin( size: '0', }, }); - const writeStream = createWriteStream(exportPath); - const zip = new Zip(); - const onBackpressure = (stream: any, outputStream: any, cb: any) => { - const runCb = () => { - cb(applyOutputBackpressure || backpressureBytes > backpressureThreshold); - }; + const zip = archiver('zip', { + zlib: { level: 9 }, + }); - const backpressureThreshold = 65536; - const backpressure: number[] = []; - let backpressureBytes = 0; - const push = stream.push; - stream.push = (data: string | any[], final: any) => { - backpressure.push(data.length); - backpressureBytes += data.length; - runCb(); - push.call(stream, data, final); - }; - let ondata = stream.ondata; - const ondataPatched = (err: any, data: any, final: any) => { - ondata.call(stream, err, data, final); - backpressureBytes -= backpressure.shift()!; - runCb(); - }; - if (ondata) { - stream.ondata = ondataPatched; - } else { - Object.defineProperty(stream, 'ondata', { - get: () => ondataPatched, - set: (cb) => (ondata = cb), - }); + zip.pipe(writeStream); + + let totalSize = 0; + for (const file of files) { + const stream = await datasource.get(file.name); + if (!stream) { + logger.warn(`failed to get file ${file.name}`); + continue; } - let applyOutputBackpressure = false; - const write = outputStream.write; - outputStream.write = (data: any) => { - const outputNotFull = write.call(outputStream, data); - applyOutputBackpressure = !outputNotFull; - runCb(); - }; - outputStream.on('drain', () => { - applyOutputBackpressure = false; - runCb(); - }); - }; + zip.append(stream, { name: file.name }); + totalSize += file.size; + logger.debug('file added to zip', { name: file.name, size: file.size }); + } - zip.ondata = async (err, data, final) => { - if (err) { - writeStream.close(); - logger.debug('error while writing to zip', { err }); - logger.error(`export for ${req.user.id} failed`); - - await prisma.export.delete({ where: { id: exportDb.id } }); - - return; - } - - writeStream.write(data); - - if (!final) return; - - writeStream.end(); - logger.debug('exported', { path: exportPath, bytes: data.length }); + writeStream.on('close', async () => { + logger.debug('exported', { path: exportPath, bytes: zip.pointer() }); logger.info(`export for ${req.user.id} finished at ${exportPath}`); await prisma.export.update({ @@ -165,37 +125,15 @@ export default fastifyPlugin( size: (await stat(exportPath)).size.toString(), }, }); - }; + }); - for (let i = 0; i !== files.length; ++i) { - const file = files[i]; + zip.on('error', (err) => { + logger.error('export zip error', { err, exportId: exportDb.id }); + }); - const stream = await datasource.get(file.name); - if (!stream) { - logger.warn(`failed to get file ${file.name}`); - continue; - } + zip.finalize(); - const passThrough = new ZipPassThrough(file.name); - zip.add(passThrough); - - onBackpressure(passThrough, stream, (applyBackpressure: boolean) => { - if (applyBackpressure) { - stream.pause(); - } else if (stream.isPaused()) { - stream.resume(); - } - }); - stream.on('data', (c) => passThrough.push(c)); - stream.on('end', () => { - passThrough.push(new Uint8Array(0), true); - logger.debug(`file ${i + 1}/${files.length} added to zip`, { name: file.name }); - }); - } - - zip.end(); - - logger.info(`export for ${req.user.id} started`); + logger.info(`export for ${req.user.id} started`, { totalSize: bytes(totalSize) }); return res.send({ running: true }); });